@litmers/cursorflow-orchestrator 0.1.29 → 0.1.31

package/src/core/orchestrator.ts

@@ -86,6 +86,26 @@ interface RunningLaneInfo {
   continueSignalsSent: number; // Number of continue signals sent
 }
 
+/**
+ * Log the tail of a file
+ */
+function logFileTail(filePath: string, lines: number = 10): void {
+  if (!fs.existsSync(filePath)) return;
+  try {
+    const content = fs.readFileSync(filePath, 'utf8');
+    const allLines = content.split('\n');
+    const tail = allLines.slice(-lines).filter(l => l.trim());
+    if (tail.length > 0) {
+      logger.error(`  Last ${tail.length} lines of log:`);
+      for (const line of tail) {
+        logger.error(`    ${line}`);
+      }
+    }
+  } catch (e) {
+    // Ignore log reading errors
+  }
+}
+
 /**
  * Spawn a lane process
  */
@@ -189,9 +209,12 @@ export function spawnLane({
           // or if it's NOT a noisy JSON line
           const hasTimestamp = /^\[\d{4}-\d{2}-\d{2}T|\^\[\d{2}:\d{2}:\d{2}\]/.test(trimmed);
           const isJson = trimmed.startsWith('{') || trimmed.includes('{"type"');
+          // Filter out heartbeats - they should NOT reset the idle timer
+          const isHeartbeat = trimmed.includes('Heartbeat') && trimmed.includes('bytes received');
           
           if (trimmed && !isJson) {
-            if (onActivity) onActivity();
+            // Only trigger activity for non-heartbeat lines
+            if (onActivity && !isHeartbeat) onActivity();
             // If line alreedy has timestamp format, just add lane prefix
             if (hasTimestamp) {
               // Insert lane name after first timestamp
@@ -651,7 +674,28 @@ export async function orchestrate(tasksDir: string, options: {
     printLaneStatus(lanes, laneRunDirs);
   }, options.pollInterval || 60000);
   
-  while (completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length || (blockedLanes.size > 0 && running.size === 0)) {
+  // Handle process interruption
+  const sigIntHandler = () => {
+    logger.warn('\n⚠️  Orchestration interrupted! Stopping all lanes...');
+    for (const [name, info] of running.entries()) {
+      logger.info(`Stopping lane: ${name}`);
+      try {
+        info.child.kill('SIGTERM');
+      } catch {
+        // Ignore kill errors
+      }
+    }
+    printLaneStatus(lanes, laneRunDirs);
+    process.exit(130);
+  };
+  
+  process.on('SIGINT', sigIntHandler);
+  process.on('SIGTERM', sigIntHandler);
+  
+  let lastStallCheck = Date.now();
+  
+  try {
+    while (completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length || (blockedLanes.size > 0 && running.size === 0)) {
     // 1. Identify lanes ready to start
     const readyToStart = lanes.filter(lane => {
       // Not already running or completed or failed or blocked
@@ -692,6 +736,23 @@ export async function orchestrate(tasksDir: string, options: {
       
       logger.info(`Lane started: ${lane.name}${lane.startIndex ? ` (resuming from ${lane.startIndex})` : ''}`);
       
+      const now = Date.now();
+      // Pre-register lane in running map so onActivity can find it immediately
+      running.set(lane.name, {
+        child: {} as any, // Placeholder, will be replaced below
+        logManager: undefined,
+        logPath: '',
+        lastActivity: now,
+        lastStateUpdate: now,
+        stallPhase: 0,
+        taskStartTime: now,
+        lastOutput: '',
+        statePath: laneStatePath,
+        bytesReceived: 0,
+        lastBytesCheck: 0,
+        continueSignalsSent: 0,
+      });
+
       let lastOutput = '';
       const spawnResult = spawnLane({
         laneName: lane.name,
@@ -706,44 +767,49 @@ export async function orchestrate(tasksDir: string, options: {
         onActivity: () => {
           const info = running.get(lane.name);
           if (info) {
-            const now = Date.now();
-            info.lastActivity = now;
-            // Also reset progress tracking when there's activity (THNK/TOOL events)
-            // This prevents STALL_NO_PROGRESS from firing when agent is actively working
-            info.lastStateUpdate = now;
-            info.stallPhase = 0; // Reset stall phase since agent is responding
+            const actNow = Date.now();
+            info.lastActivity = actNow;
+            info.lastStateUpdate = actNow;
+            info.stallPhase = 0;
           }
         }
       });
       
+      // Update with actual spawn result
+      const existingInfo = running.get(lane.name)!;
+      Object.assign(existingInfo, spawnResult);
+      
       // Track last output and bytes received for long operation and stall detection
       if (spawnResult.child.stdout) {
         spawnResult.child.stdout.on('data', (data: Buffer) => {
           const info = running.get(lane.name);
           if (info) {
-            info.lastOutput = data.toString().trim().split('\n').pop() || '';
-            info.bytesReceived += data.length;
+            const output = data.toString();
+            const lines = output.split('\n').filter(l => l.trim());
+            
+            // Filter out heartbeats from activity tracking to avoid resetting stall detection
+            const realLines = lines.filter(line => !(line.includes('Heartbeat') && line.includes('bytes received')));
             
-            // Update auto-recovery manager
-            autoRecoveryManager.recordActivity(lane.name, data.length, info.lastOutput);
+            if (realLines.length > 0) {
+              // Real activity detected - update lastActivity to reset stall timer
+              const actNow = Date.now();
+              info.lastActivity = actNow;
+              info.stallPhase = 0; // Reset stall phase on real activity
+              
+              const lastRealLine = realLines[realLines.length - 1]!;
+              info.lastOutput = lastRealLine;
+              info.bytesReceived += data.length;
+              
+              // Update auto-recovery manager with real activity
+              autoRecoveryManager.recordActivity(lane.name, data.length, info.lastOutput);
+            } else if (lines.length > 0) {
+              // Only heartbeats received - do NOT update lastActivity (keep stall timer running)
+              autoRecoveryManager.recordActivity(lane.name, 0, info.lastOutput);
+            }
           }
         });
       }
       
-      const now = Date.now();
-      running.set(lane.name, {
-        ...spawnResult,
-        lastActivity: now,
-        lastStateUpdate: now,
-        stallPhase: 0,
-        taskStartTime: now,
-        lastOutput: '',
-        statePath: laneStatePath,
-        bytesReceived: 0,
-        lastBytesCheck: 0,
-        continueSignalsSent: 0,
-      });
-      
       // Register lane with auto-recovery manager
       autoRecoveryManager.registerLane(lane.name);
       
@@ -773,13 +839,19 @@ export async function orchestrate(tasksDir: string, options: {
       const result = await Promise.race([...promises, pollPromise]);
       if (pollTimeout) clearTimeout(pollTimeout);
       
-      if (result.name === '__poll__') {
+      const now = Date.now();
+      if (result.name === '__poll__' || (now - lastStallCheck >= 10000)) {
+        lastStallCheck = now;
+        
         // Periodic stall check with multi-layer detection and escalating recovery
         for (const [laneName, info] of running.entries()) {
-          const now = Date.now();
           const idleTime = now - info.lastActivity;
           const lane = lanes.find(l => l.name === laneName)!;
           
+          if (process.env['DEBUG_STALL']) {
+            logger.debug(`[${laneName}] Stall check: idle=${Math.round(idleTime/1000)}s, bytesDelta=${info.bytesReceived - info.lastBytesCheck}, phase=${info.stallPhase}`);
+          }
+
           // Check state file for progress updates
           let progressTime = 0;
           try {
@@ -1002,74 +1074,98 @@ export async function orchestrate(tasksDir: string, options: {
           }
         }
         continue;
-      }
-
-      const finished = result;
-      const info = running.get(finished.name)!;
-      running.delete(finished.name);
-      exitCodes[finished.name] = finished.code;
-      
-      // Unregister from auto-recovery manager
-      autoRecoveryManager.unregisterLane(finished.name);
-      
-      if (finished.code === 0) {
-        completedLanes.add(finished.name);
-        events.emit('lane.completed', {
-          laneName: finished.name,
-          exitCode: finished.code,
-        });
-      } else if (finished.code === 2) {
-        // Blocked by dependency
-        const statePath = safeJoin(laneRunDirs[finished.name]!, 'state.json');
-        const state = loadState<LaneState>(statePath);
+      } else {
+        const finished = result;
+        const info = running.get(finished.name)!;
+        running.delete(finished.name);
+        exitCodes[finished.name] = finished.code;
         
-        if (state && state.dependencyRequest) {
-          blockedLanes.set(finished.name, state.dependencyRequest);
-          const lane = lanes.find(l => l.name === finished.name);
-          if (lane) {
-            lane.startIndex = Math.max(0, state.currentTaskIndex - 1); // Task was blocked, retry it
-          }
-          
-          events.emit('lane.blocked', {
+        // Unregister from auto-recovery manager
+        autoRecoveryManager.unregisterLane(finished.name);
+        
+        if (finished.code === 0) {
+          completedLanes.add(finished.name);
+          events.emit('lane.completed', {
             laneName: finished.name,
-            dependencyRequest: state.dependencyRequest,
+            exitCode: finished.code,
           });
-          logger.warn(`Lane ${finished.name} is blocked on dependency change request`);
-        } else {
-          failedLanes.add(finished.name);
-          logger.error(`Lane ${finished.name} exited with code 2 but no dependency request found`);
-        }
-      } else {
-        // Check if it was a restart request
-        if (info.stallPhase === 2) {
-          logger.info(`🔄 Lane ${finished.name} is being restarted due to stall...`);
-          
-          // Update startIndex from current state to resume from the same task
+        } else if (finished.code === 2) {
+          // Blocked by dependency
           const statePath = safeJoin(laneRunDirs[finished.name]!, 'state.json');
           const state = loadState<LaneState>(statePath);
-          if (state) {
+          
+          if (state && state.dependencyRequest) {
+            blockedLanes.set(finished.name, state.dependencyRequest);
             const lane = lanes.find(l => l.name === finished.name);
             if (lane) {
-              lane.startIndex = state.currentTaskIndex;
+              lane.startIndex = Math.max(0, state.currentTaskIndex - 1); // Task was blocked, retry it
+            }
+            
+            events.emit('lane.blocked', {
+              laneName: finished.name,
+              dependencyRequest: state.dependencyRequest,
+            });
+            logger.warn(`Lane ${finished.name} is blocked on dependency change request`);
+          } else {
+            failedLanes.add(finished.name);
+            logger.error(`Lane ${finished.name} exited with code 2 but no dependency request found`);
+          }
+        } else {
+          // Check if it was a restart request
+          if (info.stallPhase === 2) {
+            logger.info(`🔄 Lane ${finished.name} is being restarted due to stall...`);
+            
+            // Update startIndex from current state to resume from the same task
+            const statePath = safeJoin(laneRunDirs[finished.name]!, 'state.json');
+            const state = loadState<LaneState>(statePath);
+            if (state) {
+              const lane = lanes.find(l => l.name === finished.name);
+              if (lane) {
+                lane.startIndex = state.currentTaskIndex;
+              }
             }
+            
+            // Note: we don't add to failedLanes or completedLanes, 
+            // so it will be eligible to start again in the next iteration.
+            continue; 
           }
+
+          failedLanes.add(finished.name);
           
-          // Note: we don't add to failedLanes or completedLanes, 
-          // so it will be eligible to start again in the next iteration.
-          continue; 
-        }
+          let errorMsg = 'Process exited with non-zero code';
+          if (info.stallPhase === 3) {
+            errorMsg = 'Stopped due to repeated stall';
+          } else if (info.logManager) {
+            const lastError = info.logManager.getLastError();
+            if (lastError) {
+              errorMsg = `Process failed: ${lastError}`;
+            }
+          }
 
-        failedLanes.add(finished.name);
-        events.emit('lane.failed', {
-          laneName: finished.name,
-          exitCode: finished.code,
-          error: info.stallPhase === 3 ? 'Stopped due to repeated stall' : 'Process exited with non-zero code',
-        });
+          logger.error(`[${finished.name}] Lane failed with exit code ${finished.code}: ${errorMsg}`);
+          
+          // Log log tail for visibility
+          if (info.logPath) {
+            logFileTail(info.logPath, 15);
+          }
+
+          events.emit('lane.failed', {
+            laneName: finished.name,
+            exitCode: finished.code,
+            error: errorMsg,
+          });
+        }
+        
+        printLaneStatus(lanes, laneRunDirs);
       }
-      
-      printLaneStatus(lanes, laneRunDirs);
     } else {
       // Nothing running. Are we blocked?
+      
+      // Wait a bit to avoid busy-spin while waiting for dependencies or new slots
+      if (completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length) {
+        await new Promise(resolve => setTimeout(resolve, 2000));
+      }
+
       if (blockedLanes.size > 0 && autoResolve) {
         logger.section('🛠 Auto-Resolving Dependencies');
         
@@ -1098,10 +1194,14 @@ export async function orchestrate(tasksDir: string, options: {
         // All finished
         break;
       }
+      }
     }
+  } finally {
+    clearInterval(monitorInterval);
+    process.removeListener('SIGINT', sigIntHandler);
+    process.removeListener('SIGTERM', sigIntHandler);
   }
   
-  clearInterval(monitorInterval);
   printLaneStatus(lanes, laneRunDirs);
   
   // Check for failures

package/src/utils/config.ts

@@ -6,6 +6,7 @@
 
 import * as path from 'path';
 import * as fs from 'fs';
+import * as git from './git';
 import { CursorFlowConfig } from './types';
 import { safeJoin } from './path';
 export { CursorFlowConfig };
@@ -167,6 +168,7 @@ export function validateConfig(config: CursorFlowConfig): boolean {
  */
 export function createDefaultConfig(projectRoot: string, force = false): string {
   const configPath = safeJoin(projectRoot, 'cursorflow.config.js');
+  const currentBranch = git.getCurrentBranch() || 'main';
   
   const template = `module.exports = {
   // Directory configuration
@@ -175,7 +177,7 @@ export function createDefaultConfig(projectRoot: string, force = false): string
   pofDir: '_cursorflow/pof',
   
   // Git configuration
-  baseBranch: git.getCurrentBranch() || 'main',
+  baseBranch: '${currentBranch}',
   branchPrefix: 'feature/',
   
   // Execution configuration

package/src/utils/enhanced-logger.ts

@@ -1024,7 +1024,7 @@ export class EnhancedLogManager {
   }
 
   /**
-   * Close all log files
+   * Close all log files and ensure all data is flushed to disk
    */
   public close(): void {
     // Flush transform stream
@@ -1051,46 +1051,87 @@ export class EnhancedLogManager {
 `;
     
     if (this.cleanLogFd !== null) {
-      fs.writeSync(this.cleanLogFd, endMarker);
-      fs.closeSync(this.cleanLogFd);
+      try {
+        fs.writeSync(this.cleanLogFd, endMarker);
+        fs.fsyncSync(this.cleanLogFd);
+        fs.closeSync(this.cleanLogFd);
+      } catch {}
       this.cleanLogFd = null;
     }
     
     if (this.rawLogFd !== null) {
-      fs.writeSync(this.rawLogFd, endMarker);
-      fs.closeSync(this.rawLogFd);
+      try {
+        fs.writeSync(this.rawLogFd, endMarker);
+        fs.fsyncSync(this.rawLogFd);
+        fs.closeSync(this.rawLogFd);
+      } catch {}
       this.rawLogFd = null;
     }
     
     if (this.absoluteRawLogFd !== null) {
-      fs.closeSync(this.absoluteRawLogFd);
+      try {
+        fs.fsyncSync(this.absoluteRawLogFd);
+        fs.closeSync(this.absoluteRawLogFd);
+      } catch {}
       this.absoluteRawLogFd = null;
     }
     
     if (this.jsonLogFd !== null) {
-      this.writeJsonEntry({
-        timestamp: new Date().toISOString(),
-        level: 'session',
-        source: 'system',
-        lane: this.session.laneName,
-        message: 'Session ended',
-        metadata: {
-          sessionId: this.session.id,
-          duration: Date.now() - this.session.startTime,
-        },
-      });
-      fs.closeSync(this.jsonLogFd);
+      try {
+        this.writeJsonEntry({
+          timestamp: new Date().toISOString(),
+          level: 'session',
+          source: 'system',
+          lane: this.session.laneName,
+          message: 'Session ended',
+          metadata: {
+            sessionId: this.session.id,
+            duration: Date.now() - this.session.startTime,
+          },
+        });
+        fs.fsyncSync(this.jsonLogFd);
+        fs.closeSync(this.jsonLogFd);
+      } catch {}
       this.jsonLogFd = null;
     }
     
     // Close readable log
     if (this.readableLogFd !== null) {
-      fs.writeSync(this.readableLogFd, endMarker);
-      fs.closeSync(this.readableLogFd);
+      try {
+        fs.writeSync(this.readableLogFd, endMarker);
+        fs.fsyncSync(this.readableLogFd);
+        fs.closeSync(this.readableLogFd);
+      } catch {}
       this.readableLogFd = null;
     }
   }
 
+  /**
+   * Extract the last error message from the clean log file
+   */
+  public getLastError(): string | null {
+    try {
+      if (!fs.existsSync(this.cleanLogPath)) return null;
+      const content = fs.readFileSync(this.cleanLogPath, 'utf8');
+      // Look for lines containing error markers
+      const lines = content.split('\n').filter(l => 
+        l.includes('[ERROR]') || 
+        l.includes('❌') || 
+        l.includes('error:') || 
+        l.includes('Fatal') ||
+        l.includes('fail')
+      );
+      if (lines.length === 0) {
+        // Fallback to last 5 lines if no specific error marker found
+        const allLines = content.split('\n').filter(l => l.trim());
+        return allLines.slice(-5).join('\n');
+      }
+      return lines[lines.length - 1]!.trim();
+    } catch {
+      return null;
+    }
+  }
+
   /**
    * Format duration for display
    */

package/src/utils/git.ts

@@ -132,6 +132,10 @@ function filterGitStderr(stderr: string): string {
 export function runGit(args: string[], options: GitRunOptions = {}): string {
   const { cwd, silent = false } = options;
   
+  if (process.env['DEBUG_GIT']) {
+    console.log(`[DEBUG_GIT] Running: git ${args.join(' ')} (cwd: ${cwd || process.cwd()})`);
+  }
+  
   try {
     const stdioMode = silent ? 'pipe' : ['inherit', 'inherit', 'pipe'];
 
@@ -141,6 +145,13 @@ export function runGit(args: string[], options: GitRunOptions = {}): string {
       stdio: stdioMode as any,
     });
     
+    if (result.error) {
+      if (!silent) {
+        console.error(`[ERROR_GIT] Failed to execute git command: ${result.error.message}`);
+      }
+      throw result.error;
+    }
+
     if (!silent && result.stderr) {
       const filteredStderr = filterGitStderr(result.stderr);
       if (filteredStderr) {
@@ -149,12 +160,16 @@ export function runGit(args: string[], options: GitRunOptions = {}): string {
     }
     
     if (result.status !== 0 && !silent) {
-      throw new Error(`Git command failed: git ${args.join(' ')}\n${result.stderr || ''}`);
+      const errorMsg = `Git command failed (exit code ${result.status}): git ${args.join(' ')}\n${result.stderr || ''}`;
+      throw new Error(errorMsg);
     }
     
     return result.stdout ? result.stdout.trim() : '';
   } catch (error) {
     if (silent) {
+      if (process.env['DEBUG_GIT']) {
+        console.error(`[DEBUG_GIT] Command failed (silent mode): ${error instanceof Error ? error.message : String(error)}`);
+      }
       return '';
     }
     throw error;
@@ -167,18 +182,28 @@ export function runGit(args: string[], options: GitRunOptions = {}): string {
 export function runGitResult(args: string[], options: GitRunOptions = {}): GitResult {
   const { cwd } = options;
   
+  if (process.env['DEBUG_GIT']) {
+    console.log(`[DEBUG_GIT] Running: git ${args.join(' ')} (result mode, cwd: ${cwd || process.cwd()})`);
+  }
+
   const result = spawnSync('git', args, {
     cwd: cwd || process.cwd(),
     encoding: 'utf8',
     stdio: 'pipe',
   });
   
-  return {
+  const gitResult = {
     exitCode: result.status ?? 1,
     stdout: (result.stdout || '').toString().trim(),
     stderr: (result.stderr || '').toString().trim(),
     success: result.status === 0,
   };
+
+  if (process.env['DEBUG_GIT'] && !gitResult.success) {
+    console.error(`[DEBUG_GIT] Command failed: git ${args.join(' ')}\nstderr: ${gitResult.stderr}`);
+  }
+
+  return gitResult;
 }
 
 /**
@@ -829,6 +854,94 @@ export function getWorktreeForPath(targetPath: string, cwd?: string): WorktreeIn
 /**
  * Sync branch with remote (fetch + merge or rebase)
  */
+/**
+ * Push branch to remote, creating it if it doesn't exist
+ * Returns success status and any error message
+ */
+export function pushBranchSafe(branchName: string, options: { cwd?: string; force?: boolean } = {}): { success: boolean; error?: string } {
+  const { cwd, force = false } = options;
+  
+  // Check if origin exists
+  if (!remoteExists('origin', { cwd })) {
+    return { success: false, error: 'No remote "origin" configured' };
+  }
+  
+  const args = ['push'];
+  if (force) {
+    args.push('--force');
+  }
+  args.push('-u', 'origin', branchName);
+  
+  const result = runGitResult(args, { cwd });
+  
+  if (result.success) {
+    return { success: true };
+  }
+  
+  return { success: false, error: result.stderr };
+}
+
+/**
+ * Auto-commit any uncommitted changes and push to remote
+ * Used for checkpoint before destructive operations
+ */
+export function checkpointAndPush(options: { 
+  cwd?: string; 
+  message?: string;
+  branchName?: string;
+} = {}): { success: boolean; committed: boolean; pushed: boolean; error?: string } {
+  const { cwd, message = '[cursorflow] checkpoint before clean' } = options;
+  
+  let committed = false;
+  let pushed = false;
+  
+  // Get current branch if not specified
+  let branchName = options.branchName;
+  if (!branchName) {
+    try {
+      branchName = getCurrentBranch(cwd);
+    } catch {
+      return { success: false, committed: false, pushed: false, error: 'Failed to get current branch' };
+    }
+  }
+  
+  // Check for uncommitted changes
+  if (hasUncommittedChanges(cwd)) {
+    // Stage all changes
+    const addResult = runGitResult(['add', '-A'], { cwd });
+    if (!addResult.success) {
+      return { success: false, committed: false, pushed: false, error: `Failed to stage changes: ${addResult.stderr}` };
+    }
+    
+    // Commit
+    const commitResult = runGitResult(['commit', '-m', message], { cwd });
+    if (!commitResult.success) {
+      return { success: false, committed: false, pushed: false, error: `Failed to commit: ${commitResult.stderr}` };
+    }
+    committed = true;
+  }
+  
+  // Push to remote
+  const pushResult = pushBranchSafe(branchName, { cwd });
+  if (pushResult.success) {
+    pushed = true;
+  } else {
+    // Push failed but commit succeeded - partial success
+    if (committed) {
+      return { success: true, committed: true, pushed: false, error: `Commit succeeded but push failed: ${pushResult.error}` };
+    }
+    // Nothing to commit and push failed - check if there's anything to push
+    const localCommits = runGitResult(['rev-list', `origin/${branchName}..HEAD`], { cwd });
+    if (localCommits.success && localCommits.stdout.trim()) {
+      return { success: false, committed: false, pushed: false, error: `Push failed: ${pushResult.error}` };
+    }
+    // Nothing to push
+    pushed = true;
+  }
+  
+  return { success: true, committed, pushed };
+}
+
 export function syncWithRemote(branch: string, options: { 
   cwd?: string; 
   strategy?: 'merge' | 'rebase';