@litmers/cursorflow-orchestrator 0.1.29 → 0.1.31

package/dist/core/orchestrator.js

@@ -73,6 +73,27 @@ const DEFAULT_ORCHESTRATOR_STALL_CONFIG = {
     progressTimeoutMs: 10 * 60 * 1000, // 10 minutes (only triggers if no activity at all)
     maxRestarts: 2,
 };
+/**
+ * Log the tail of a file
+ */
+function logFileTail(filePath, lines = 10) {
+    if (!fs.existsSync(filePath))
+        return;
+    try {
+        const content = fs.readFileSync(filePath, 'utf8');
+        const allLines = content.split('\n');
+        const tail = allLines.slice(-lines).filter(l => l.trim());
+        if (tail.length > 0) {
+            logger.error(`  Last ${tail.length} lines of log:`);
+            for (const line of tail) {
+                logger.error(`    ${line}`);
+            }
+        }
+    }
+    catch (e) {
+        // Ignore log reading errors
+    }
+}
 /**
  * Spawn a lane process
  */
@@ -141,8 +162,11 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
                     // or if it's NOT a noisy JSON line
                     const hasTimestamp = /^\[\d{4}-\d{2}-\d{2}T|\^\[\d{2}:\d{2}:\d{2}\]/.test(trimmed);
                     const isJson = trimmed.startsWith('{') || trimmed.includes('{"type"');
+                    // Filter out heartbeats - they should NOT reset the idle timer
+                    const isHeartbeat = trimmed.includes('Heartbeat') && trimmed.includes('bytes received');
                     if (trimmed && !isJson) {
-                        if (onActivity)
+                        // Only trigger activity for non-heartbeat lines
+                        if (onActivity && !isHeartbeat)
                             onActivity();
                         // If line alreedy has timestamp format, just add lane prefix
                         if (hasTimestamp) {
@@ -524,417 +548,485 @@ async function orchestrate(tasksDir, options = {}) {
     const monitorInterval = setInterval(() => {
         printLaneStatus(lanes, laneRunDirs);
     }, options.pollInterval || 60000);
-    while (completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length || (blockedLanes.size > 0 && running.size === 0)) {
-        // 1. Identify lanes ready to start
-        const readyToStart = lanes.filter(lane => {
-            // Not already running or completed or failed or blocked
-            if (running.has(lane.name) || completedLanes.has(lane.name) || failedLanes.has(lane.name) || blockedLanes.has(lane.name)) {
-                return false;
+    // Handle process interruption
+    const sigIntHandler = () => {
+        logger.warn('\n⚠️  Orchestration interrupted! Stopping all lanes...');
+        for (const [name, info] of running.entries()) {
+            logger.info(`Stopping lane: ${name}`);
+            try {
+                info.child.kill('SIGTERM');
             }
-            // Check dependencies
-            for (const dep of lane.dependsOn) {
-                if (failedLanes.has(dep)) {
-                    logger.error(`Lane ${lane.name} will not start because dependency ${dep} failed`);
-                    failedLanes.add(lane.name);
-                    exitCodes[lane.name] = 1;
-                    return false;
-                }
-                if (blockedLanes.has(dep)) {
-                    // If a dependency is blocked, wait
-                    return false;
-                }
-                if (!completedLanes.has(dep)) {
+            catch {
+                // Ignore kill errors
+            }
+        }
+        printLaneStatus(lanes, laneRunDirs);
+        process.exit(130);
+    };
+    process.on('SIGINT', sigIntHandler);
+    process.on('SIGTERM', sigIntHandler);
+    let lastStallCheck = Date.now();
+    try {
+        while (completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length || (blockedLanes.size > 0 && running.size === 0)) {
+            // 1. Identify lanes ready to start
+            const readyToStart = lanes.filter(lane => {
+                // Not already running or completed or failed or blocked
+                if (running.has(lane.name) || completedLanes.has(lane.name) || failedLanes.has(lane.name) || blockedLanes.has(lane.name)) {
                     return false;
                 }
-            }
-            return true;
-        });
-        // 2. Spawn ready lanes up to maxConcurrent
-        for (const lane of readyToStart) {
-            if (running.size >= maxConcurrent)
-                break;
-            const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[lane.name], 'state.json');
-            // Validate and repair state before starting
-            const validation = (0, state_1.validateLaneState)(laneStatePath, { autoRepair: true });
-            if (!validation.valid && !validation.repaired) {
-                logger.warn(`[${lane.name}] State validation issues: ${validation.issues.join(', ')}`);
-            }
-            logger.info(`Lane started: ${lane.name}${lane.startIndex ? ` (resuming from ${lane.startIndex})` : ''}`);
-            let lastOutput = '';
-            const spawnResult = spawnLane({
-                laneName: lane.name,
-                tasksFile: lane.path,
-                laneRunDir: laneRunDirs[lane.name],
-                executor: options.executor || 'cursor-agent',
-                startIndex: lane.startIndex,
-                pipelineBranch: `${pipelineBranch}/${lane.name}`,
-                worktreeDir: laneWorktreeDirs[lane.name],
-                enhancedLogConfig: options.enhancedLogging,
-                noGit: options.noGit,
-                onActivity: () => {
-                    const info = running.get(lane.name);
-                    if (info) {
-                        const now = Date.now();
-                        info.lastActivity = now;
-                        // Also reset progress tracking when there's activity (THNK/TOOL events)
-                        // This prevents STALL_NO_PROGRESS from firing when agent is actively working
-                        info.lastStateUpdate = now;
-                        info.stallPhase = 0; // Reset stall phase since agent is responding
+                // Check dependencies
+                for (const dep of lane.dependsOn) {
+                    if (failedLanes.has(dep)) {
+                        logger.error(`Lane ${lane.name} will not start because dependency ${dep} failed`);
+                        failedLanes.add(lane.name);
+                        exitCodes[lane.name] = 1;
+                        return false;
+                    }
+                    if (blockedLanes.has(dep)) {
+                        // If a dependency is blocked, wait
+                        return false;
+                    }
+                    if (!completedLanes.has(dep)) {
+                        return false;
                     }
                 }
+                return true;
             });
-            // Track last output and bytes received for long operation and stall detection
-            if (spawnResult.child.stdout) {
-                spawnResult.child.stdout.on('data', (data) => {
-                    const info = running.get(lane.name);
-                    if (info) {
-                        info.lastOutput = data.toString().trim().split('\n').pop() || '';
-                        info.bytesReceived += data.length;
-                        // Update auto-recovery manager
-                        autoRecoveryManager.recordActivity(lane.name, data.length, info.lastOutput);
-                    }
+            // 2. Spawn ready lanes up to maxConcurrent
+            for (const lane of readyToStart) {
+                if (running.size >= maxConcurrent)
+                    break;
+                const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[lane.name], 'state.json');
+                // Validate and repair state before starting
+                const validation = (0, state_1.validateLaneState)(laneStatePath, { autoRepair: true });
+                if (!validation.valid && !validation.repaired) {
+                    logger.warn(`[${lane.name}] State validation issues: ${validation.issues.join(', ')}`);
+                }
+                logger.info(`Lane started: ${lane.name}${lane.startIndex ? ` (resuming from ${lane.startIndex})` : ''}`);
+                const now = Date.now();
+                // Pre-register lane in running map so onActivity can find it immediately
+                running.set(lane.name, {
+                    child: {}, // Placeholder, will be replaced below
+                    logManager: undefined,
+                    logPath: '',
+                    lastActivity: now,
+                    lastStateUpdate: now,
+                    stallPhase: 0,
+                    taskStartTime: now,
+                    lastOutput: '',
+                    statePath: laneStatePath,
+                    bytesReceived: 0,
+                    lastBytesCheck: 0,
+                    continueSignalsSent: 0,
                 });
-            }
-            const now = Date.now();
-            running.set(lane.name, {
-                ...spawnResult,
-                lastActivity: now,
-                lastStateUpdate: now,
-                stallPhase: 0,
-                taskStartTime: now,
-                lastOutput: '',
-                statePath: laneStatePath,
-                bytesReceived: 0,
-                lastBytesCheck: 0,
-                continueSignalsSent: 0,
-            });
-            // Register lane with auto-recovery manager
-            autoRecoveryManager.registerLane(lane.name);
-            // Update lane tracking
-            lane.taskStartTime = now;
-            events_1.events.emit('lane.started', {
-                laneName: lane.name,
-                pid: spawnResult.child.pid,
-                logPath: spawnResult.logPath,
-            });
-        }
-        // 3. Wait for any running lane to finish OR check for stalls
-        if (running.size > 0) {
-            // Polling timeout for stall detection
-            let pollTimeout;
-            const pollPromise = new Promise(resolve => {
-                pollTimeout = setTimeout(() => resolve({ name: '__poll__', code: 0 }), 10000);
-            });
-            const promises = Array.from(running.entries()).map(async ([name, { child }]) => {
-                const code = await waitChild(child);
-                return { name, code };
-            });
-            const result = await Promise.race([...promises, pollPromise]);
-            if (pollTimeout)
-                clearTimeout(pollTimeout);
-            if (result.name === '__poll__') {
-                // Periodic stall check with multi-layer detection and escalating recovery
-                for (const [laneName, info] of running.entries()) {
-                    const now = Date.now();
-                    const idleTime = now - info.lastActivity;
-                    const lane = lanes.find(l => l.name === laneName);
-                    // Check state file for progress updates
-                    let progressTime = 0;
-                    try {
-                        const stateStat = fs.statSync(info.statePath);
-                        const stateUpdateTime = stateStat.mtimeMs;
-                        if (stateUpdateTime > info.lastStateUpdate) {
-                            info.lastStateUpdate = stateUpdateTime;
+                let lastOutput = '';
+                const spawnResult = spawnLane({
+                    laneName: lane.name,
+                    tasksFile: lane.path,
+                    laneRunDir: laneRunDirs[lane.name],
+                    executor: options.executor || 'cursor-agent',
+                    startIndex: lane.startIndex,
+                    pipelineBranch: `${pipelineBranch}/${lane.name}`,
+                    worktreeDir: laneWorktreeDirs[lane.name],
+                    enhancedLogConfig: options.enhancedLogging,
+                    noGit: options.noGit,
+                    onActivity: () => {
+                        const info = running.get(lane.name);
+                        if (info) {
+                            const actNow = Date.now();
+                            info.lastActivity = actNow;
+                            info.lastStateUpdate = actNow;
+                            info.stallPhase = 0;
                         }
-                        progressTime = now - info.lastStateUpdate;
-                    }
-                    catch {
-                        // State file might not exist yet
                     }
-                    // Calculate bytes received since last check
-                    const bytesDelta = info.bytesReceived - info.lastBytesCheck;
-                    info.lastBytesCheck = info.bytesReceived;
-                    // Use multi-layer stall analysis with enhanced context
-                    const analysis = (0, failure_policy_1.analyzeStall)({
-                        stallPhase: info.stallPhase,
-                        idleTimeMs: idleTime,
-                        progressTimeMs: progressTime,
-                        lastOutput: info.lastOutput,
-                        restartCount: lane.restartCount || 0,
-                        taskStartTimeMs: info.taskStartTime,
-                        bytesReceived: bytesDelta, // Bytes since last check
-                        continueSignalsSent: info.continueSignalsSent,
-                    }, stallConfig);
-                    // Only act if action is not NONE
-                    if (analysis.action !== failure_policy_1.RecoveryAction.NONE) {
-                        (0, failure_policy_1.logFailure)(laneName, analysis);
-                        info.logManager?.log('error', analysis.message);
-                        if (analysis.action === failure_policy_1.RecoveryAction.CONTINUE_SIGNAL) {
-                            const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
-                            try {
-                                fs.writeFileSync(interventionPath, 'continue');
-                                info.stallPhase = 1;
-                                info.lastActivity = now;
-                                info.continueSignalsSent++;
-                                logger.info(`[${laneName}] Sent continue signal (#${info.continueSignalsSent})`);
-                                events_1.events.emit('recovery.continue_signal', {
-                                    laneName,
-                                    idleSeconds: Math.round(idleTime / 1000),
-                                    signalCount: info.continueSignalsSent,
-                                });
+                });
+                // Update with actual spawn result
+                const existingInfo = running.get(lane.name);
+                Object.assign(existingInfo, spawnResult);
+                // Track last output and bytes received for long operation and stall detection
+                if (spawnResult.child.stdout) {
+                    spawnResult.child.stdout.on('data', (data) => {
+                        const info = running.get(lane.name);
+                        if (info) {
+                            const output = data.toString();
+                            const lines = output.split('\n').filter(l => l.trim());
+                            // Filter out heartbeats from activity tracking to avoid resetting stall detection
+                            const realLines = lines.filter(line => !(line.includes('Heartbeat') && line.includes('bytes received')));
+                            if (realLines.length > 0) {
+                                // Real activity detected - update lastActivity to reset stall timer
+                                const actNow = Date.now();
+                                info.lastActivity = actNow;
+                                info.stallPhase = 0; // Reset stall phase on real activity
+                                const lastRealLine = realLines[realLines.length - 1];
+                                info.lastOutput = lastRealLine;
+                                info.bytesReceived += data.length;
+                                // Update auto-recovery manager with real activity
+                                autoRecoveryManager.recordActivity(lane.name, data.length, info.lastOutput);
                             }
-                            catch (e) {
-                                logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
+                            else if (lines.length > 0) {
+                                // Only heartbeats received - do NOT update lastActivity (keep stall timer running)
+                                autoRecoveryManager.recordActivity(lane.name, 0, info.lastOutput);
                             }
                         }
-                        else if (analysis.action === failure_policy_1.RecoveryAction.STRONGER_PROMPT) {
-                            const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
-                            const strongerPrompt = `[SYSTEM INTERVENTION] You seem to be stuck. Please continue with your current task immediately. If you're waiting for something, explain what you need and proceed with what you can do now. If you've completed the task, summarize your work and finish.`;
-                            try {
-                                fs.writeFileSync(interventionPath, strongerPrompt);
-                                info.stallPhase = 2;
-                                info.lastActivity = now;
-                                logger.warn(`[${laneName}] Sent stronger prompt after continue signal failed`);
-                                events_1.events.emit('recovery.stronger_prompt', { laneName });
-                            }
-                            catch (e) {
-                                logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
-                            }
+                    });
+                }
+                // Register lane with auto-recovery manager
+                autoRecoveryManager.registerLane(lane.name);
+                // Update lane tracking
+                lane.taskStartTime = now;
+                events_1.events.emit('lane.started', {
+                    laneName: lane.name,
+                    pid: spawnResult.child.pid,
+                    logPath: spawnResult.logPath,
+                });
+            }
+            // 3. Wait for any running lane to finish OR check for stalls
+            if (running.size > 0) {
+                // Polling timeout for stall detection
+                let pollTimeout;
+                const pollPromise = new Promise(resolve => {
+                    pollTimeout = setTimeout(() => resolve({ name: '__poll__', code: 0 }), 10000);
+                });
+                const promises = Array.from(running.entries()).map(async ([name, { child }]) => {
+                    const code = await waitChild(child);
+                    return { name, code };
+                });
+                const result = await Promise.race([...promises, pollPromise]);
+                if (pollTimeout)
+                    clearTimeout(pollTimeout);
+                const now = Date.now();
+                if (result.name === '__poll__' || (now - lastStallCheck >= 10000)) {
+                    lastStallCheck = now;
+                    // Periodic stall check with multi-layer detection and escalating recovery
+                    for (const [laneName, info] of running.entries()) {
+                        const idleTime = now - info.lastActivity;
+                        const lane = lanes.find(l => l.name === laneName);
+                        if (process.env['DEBUG_STALL']) {
+                            logger.debug(`[${laneName}] Stall check: idle=${Math.round(idleTime / 1000)}s, bytesDelta=${info.bytesReceived - info.lastBytesCheck}, phase=${info.stallPhase}`);
                         }
-                        else if (analysis.action === failure_policy_1.RecoveryAction.KILL_AND_RESTART ||
-                            analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE ||
-                            analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE_FROM_CHECKPOINT) {
-                            lane.restartCount = (lane.restartCount || 0) + 1;
-                            info.stallPhase = 3;
-                            // Try to get checkpoint info
-                            const checkpoint = (0, checkpoint_1.getLatestCheckpoint)(laneRunDirs[laneName]);
-                            if (checkpoint) {
-                                logger.info(`[${laneName}] Checkpoint available: ${checkpoint.id} (task ${checkpoint.taskIndex})`);
-                            }
-                            // Kill the process
-                            try {
-                                info.child.kill('SIGKILL');
-                            }
-                            catch {
-                                // Process might already be dead
+                        // Check state file for progress updates
+                        let progressTime = 0;
+                        try {
+                            const stateStat = fs.statSync(info.statePath);
+                            const stateUpdateTime = stateStat.mtimeMs;
+                            if (stateUpdateTime > info.lastStateUpdate) {
+                                info.lastStateUpdate = stateUpdateTime;
                             }
-                            logger.warn(`[${laneName}] Killing and restarting lane (restart #${lane.restartCount})`);
-                            events_1.events.emit('recovery.restart', {
-                                laneName,
-                                restartCount: lane.restartCount,
-                                maxRestarts: stallConfig.maxRestarts,
-                            });
+                            progressTime = now - info.lastStateUpdate;
                         }
-                        else if (analysis.action === failure_policy_1.RecoveryAction.RUN_DOCTOR) {
-                            info.stallPhase = 4;
-                            // Run diagnostics
-                            logger.error(`[${laneName}] Running diagnostics due to persistent failures...`);
-                            // Import health check dynamically to avoid circular dependency
-                            const { checkAgentHealth, checkAuthHealth } = await Promise.resolve().then(() => __importStar(require('../utils/health')));
-                            const [agentHealth, authHealth] = await Promise.all([
-                                checkAgentHealth(),
-                                checkAuthHealth(),
-                            ]);
-                            const issues = [];
-                            if (!agentHealth.ok)
-                                issues.push(`Agent: ${agentHealth.message}`);
-                            if (!authHealth.ok)
-                                issues.push(`Auth: ${authHealth.message}`);
-                            if (issues.length > 0) {
-                                logger.error(`[${laneName}] Diagnostic issues found:\n  ${issues.join('\n  ')}`);
-                            }
-                            else {
-                                logger.warn(`[${laneName}] No obvious issues found. The problem may be with the AI model or network.`);
-                            }
-                            // Save diagnostic to file
-                            const diagnosticPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'diagnostic.json');
-                            fs.writeFileSync(diagnosticPath, JSON.stringify({
-                                timestamp: Date.now(),
-                                agentHealthy: agentHealth.ok,
-                                authHealthy: authHealth.ok,
-                                issues,
-                                analysis,
-                            }, null, 2));
-                            // Kill the process
-                            try {
-                                info.child.kill('SIGKILL');
+                        catch {
+                            // State file might not exist yet
+                        }
+                        // Calculate bytes received since last check
+                        const bytesDelta = info.bytesReceived - info.lastBytesCheck;
+                        info.lastBytesCheck = info.bytesReceived;
+                        // Use multi-layer stall analysis with enhanced context
+                        const analysis = (0, failure_policy_1.analyzeStall)({
+                            stallPhase: info.stallPhase,
+                            idleTimeMs: idleTime,
+                            progressTimeMs: progressTime,
+                            lastOutput: info.lastOutput,
+                            restartCount: lane.restartCount || 0,
+                            taskStartTimeMs: info.taskStartTime,
+                            bytesReceived: bytesDelta, // Bytes since last check
+                            continueSignalsSent: info.continueSignalsSent,
+                        }, stallConfig);
+                        // Only act if action is not NONE
+                        if (analysis.action !== failure_policy_1.RecoveryAction.NONE) {
+                            (0, failure_policy_1.logFailure)(laneName, analysis);
+                            info.logManager?.log('error', analysis.message);
+                            if (analysis.action === failure_policy_1.RecoveryAction.CONTINUE_SIGNAL) {
+                                const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
+                                try {
+                                    fs.writeFileSync(interventionPath, 'continue');
+                                    info.stallPhase = 1;
+                                    info.lastActivity = now;
+                                    info.continueSignalsSent++;
+                                    logger.info(`[${laneName}] Sent continue signal (#${info.continueSignalsSent})`);
+                                    events_1.events.emit('recovery.continue_signal', {
+                                        laneName,
+                                        idleSeconds: Math.round(idleTime / 1000),
+                                        signalCount: info.continueSignalsSent,
+                                    });
+                                }
+                                catch (e) {
+                                    logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
+                                }
                             }
-                            catch {
-                                // Process might already be dead
+                            else if (analysis.action === failure_policy_1.RecoveryAction.STRONGER_PROMPT) {
+                                const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
+                                const strongerPrompt = `[SYSTEM INTERVENTION] You seem to be stuck. Please continue with your current task immediately. If you're waiting for something, explain what you need and proceed with what you can do now. If you've completed the task, summarize your work and finish.`;
+                                try {
+                                    fs.writeFileSync(interventionPath, strongerPrompt);
+                                    info.stallPhase = 2;
+                                    info.lastActivity = now;
+                                    logger.warn(`[${laneName}] Sent stronger prompt after continue signal failed`);
+                                    events_1.events.emit('recovery.stronger_prompt', { laneName });
+                                }
+                                catch (e) {
+                                    logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
+                                }
                             }
-                            logger.error(`[${laneName}] Aborting lane after diagnostic. Check ${diagnosticPath} for details.`);
-                            // Save POF for failed recovery
-                            const recoveryState = autoRecoveryManager.getState(laneName);
-                            if (recoveryState) {
+                            else if (analysis.action === failure_policy_1.RecoveryAction.KILL_AND_RESTART ||
+                                analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE ||
+                                analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE_FROM_CHECKPOINT) {
+                                lane.restartCount = (lane.restartCount || 0) + 1;
+                                info.stallPhase = 3;
+                                // Try to get checkpoint info
+                                const checkpoint = (0, checkpoint_1.getLatestCheckpoint)(laneRunDirs[laneName]);
+                                if (checkpoint) {
+                                    logger.info(`[${laneName}] Checkpoint available: ${checkpoint.id} (task ${checkpoint.taskIndex})`);
+                                }
+                                // Kill the process
                                 try {
-                                    const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
-                                    const laneState = (0, state_1.loadState)(laneStatePath);
-                                    const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
-                                    const diagnosticInfo = {
-                                        timestamp: Date.now(),
-                                        agentHealthy: agentHealth.ok,
-                                        authHealthy: authHealth.ok,
-                                        systemHealthy: true,
-                                        suggestedAction: issues.length > 0 ? 'Fix the issues above and retry' : 'Try with a different model',
-                                        details: issues.join('\n') || 'No obvious issues found',
-                                    };
-                                    const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, diagnosticInfo);
-                                    (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
+                                    info.child.kill('SIGKILL');
                                 }
-                                catch (pofError) {
-                                    logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
+                                catch {
+                                    // Process might already be dead
                                 }
+                                logger.warn(`[${laneName}] Killing and restarting lane (restart #${lane.restartCount})`);
+                                events_1.events.emit('recovery.restart', {
+                                    laneName,
+                                    restartCount: lane.restartCount,
+                                    maxRestarts: stallConfig.maxRestarts,
+                                });
                             }
-                            events_1.events.emit('recovery.diagnosed', {
-                                laneName,
-                                diagnostic: { agentHealthy: agentHealth.ok, authHealthy: authHealth.ok, issues },
-                            });
-                        }
-                        else if (analysis.action === failure_policy_1.RecoveryAction.ABORT_LANE) {
-                            info.stallPhase = 5;
-                            try {
-                                info.child.kill('SIGKILL');
+                            else if (analysis.action === failure_policy_1.RecoveryAction.RUN_DOCTOR) {
+                                info.stallPhase = 4;
+                                // Run diagnostics
+                                logger.error(`[${laneName}] Running diagnostics due to persistent failures...`);
+                                // Import health check dynamically to avoid circular dependency
+                                const { checkAgentHealth, checkAuthHealth } = await Promise.resolve().then(() => __importStar(require('../utils/health')));
+                                const [agentHealth, authHealth] = await Promise.all([
+                                    checkAgentHealth(),
+                                    checkAuthHealth(),
+                                ]);
+                                const issues = [];
+                                if (!agentHealth.ok)
+                                    issues.push(`Agent: ${agentHealth.message}`);
+                                if (!authHealth.ok)
+                                    issues.push(`Auth: ${authHealth.message}`);
+                                if (issues.length > 0) {
+                                    logger.error(`[${laneName}] Diagnostic issues found:\n  ${issues.join('\n  ')}`);
+                                }
+                                else {
+                                    logger.warn(`[${laneName}] No obvious issues found. The problem may be with the AI model or network.`);
+                                }
+                                // Save diagnostic to file
+                                const diagnosticPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'diagnostic.json');
+                                fs.writeFileSync(diagnosticPath, JSON.stringify({
+                                    timestamp: Date.now(),
+                                    agentHealthy: agentHealth.ok,
+                                    authHealthy: authHealth.ok,
+                                    issues,
+                                    analysis,
+                                }, null, 2));
+                                // Kill the process
+                                try {
+                                    info.child.kill('SIGKILL');
+                                }
+                                catch {
+                                    // Process might already be dead
+                                }
+                                logger.error(`[${laneName}] Aborting lane after diagnostic. Check ${diagnosticPath} for details.`);
+                                // Save POF for failed recovery
+                                const recoveryState = autoRecoveryManager.getState(laneName);
+                                if (recoveryState) {
+                                    try {
+                                        const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
+                                        const laneState = (0, state_1.loadState)(laneStatePath);
+                                        const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
+                                        const diagnosticInfo = {
+                                            timestamp: Date.now(),
+                                            agentHealthy: agentHealth.ok,
+                                            authHealthy: authHealth.ok,
+                                            systemHealthy: true,
+                                            suggestedAction: issues.length > 0 ? 'Fix the issues above and retry' : 'Try with a different model',
+                                            details: issues.join('\n') || 'No obvious issues found',
+                                        };
+                                        const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, diagnosticInfo);
+                                        (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
+                                    }
+                                    catch (pofError) {
+                                        logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
+                                    }
+                                }
+                                events_1.events.emit('recovery.diagnosed', {
+                                    laneName,
+                                    diagnostic: { agentHealthy: agentHealth.ok, authHealthy: authHealth.ok, issues },
+                                });
                             }
-                            catch {
-                                // Process might already be dead
+                            else if (analysis.action === failure_policy_1.RecoveryAction.ABORT_LANE) {
+                                info.stallPhase = 5;
+                                try {
+                                    info.child.kill('SIGKILL');
+                                }
+                                catch {
+                                    // Process might already be dead
+                                }
+                                logger.error(`[${laneName}] Aborting lane due to repeated stalls`);
+                                // Save POF for failed recovery
+                                const recoveryState = autoRecoveryManager.getState(laneName);
+                                if (recoveryState) {
+                                    try {
+                                        const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
+                                        const laneState = (0, state_1.loadState)(laneStatePath);
+                                        const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
+                                        const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, recoveryState.diagnosticInfo);
+                                        (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
+                                    }
+                                    catch (pofError) {
+                                        logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
+                                    }
+                                }
                             }
-                            logger.error(`[${laneName}] Aborting lane due to repeated stalls`);
-                            // Save POF for failed recovery
-                            const recoveryState = autoRecoveryManager.getState(laneName);
-                            if (recoveryState) {
+                            else if (analysis.action === failure_policy_1.RecoveryAction.SEND_GIT_GUIDANCE) {
+                                // Send guidance message to agent for git issues
+                                const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
+                                // Determine which guidance to send based on the failure type
+                                let guidance;
+                                if (analysis.type === failure_policy_1.FailureType.GIT_PUSH_REJECTED) {
+                                    guidance = (0, auto_recovery_1.getGitPushFailureGuidance)();
+                                }
+                                else if (analysis.type === failure_policy_1.FailureType.MERGE_CONFLICT) {
+                                    guidance = (0, auto_recovery_1.getMergeConflictGuidance)();
+                                }
+                                else {
+                                    guidance = (0, auto_recovery_1.getGitErrorGuidance)(analysis.message);
+                                }
                                 try {
-                                    const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
-                                    const laneState = (0, state_1.loadState)(laneStatePath);
-                                    const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
-                                    const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, recoveryState.diagnosticInfo);
-                                    (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
+                                    fs.writeFileSync(interventionPath, guidance);
+                                    info.lastActivity = now;
+                                    logger.info(`[${laneName}] Sent git issue guidance to agent`);
                                 }
-                                catch (pofError) {
-                                    logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
+                                catch (e) {
+                                    logger.error(`[${laneName}] Failed to send guidance: ${e.message}`);
                                 }
                             }
                         }
-                        else if (analysis.action === failure_policy_1.RecoveryAction.SEND_GIT_GUIDANCE) {
-                            // Send guidance message to agent for git issues
-                            const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
-                            // Determine which guidance to send based on the failure type
-                            let guidance;
-                            if (analysis.type === failure_policy_1.FailureType.GIT_PUSH_REJECTED) {
-                                guidance = (0, auto_recovery_1.getGitPushFailureGuidance)();
-                            }
-                            else if (analysis.type === failure_policy_1.FailureType.MERGE_CONFLICT) {
-                                guidance = (0, auto_recovery_1.getMergeConflictGuidance)();
-                            }
-                            else {
-                                guidance = (0, auto_recovery_1.getGitErrorGuidance)(analysis.message);
+                    }
+                    continue;
+                }
+                else {
+                    const finished = result;
+                    const info = running.get(finished.name);
+                    running.delete(finished.name);
+                    exitCodes[finished.name] = finished.code;
+                    // Unregister from auto-recovery manager
+                    autoRecoveryManager.unregisterLane(finished.name);
+                    if (finished.code === 0) {
+                        completedLanes.add(finished.name);
+                        events_1.events.emit('lane.completed', {
+                            laneName: finished.name,
+                            exitCode: finished.code,
+                        });
+                    }
+                    else if (finished.code === 2) {
+                        // Blocked by dependency
+                        const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
+                        const state = (0, state_1.loadState)(statePath);
+                        if (state && state.dependencyRequest) {
+                            blockedLanes.set(finished.name, state.dependencyRequest);
+                            const lane = lanes.find(l => l.name === finished.name);
+                            if (lane) {
+                                lane.startIndex = Math.max(0, state.currentTaskIndex - 1); // Task was blocked, retry it
                             }
-                            try {
-                                fs.writeFileSync(interventionPath, guidance);
-                                info.lastActivity = now;
-                                logger.info(`[${laneName}] Sent git issue guidance to agent`);
+                            events_1.events.emit('lane.blocked', {
+                                laneName: finished.name,
+                                dependencyRequest: state.dependencyRequest,
+                            });
+                            logger.warn(`Lane ${finished.name} is blocked on dependency change request`);
+                        }
+                        else {
+                            failedLanes.add(finished.name);
+                            logger.error(`Lane ${finished.name} exited with code 2 but no dependency request found`);
+                        }
+                    }
+                    else {
+                        // Check if it was a restart request
+                        if (info.stallPhase === 2) {
+                            logger.info(`🔄 Lane ${finished.name} is being restarted due to stall...`);
+                            // Update startIndex from current state to resume from the same task
+                            const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
+                            const state = (0, state_1.loadState)(statePath);
+                            if (state) {
+                                const lane = lanes.find(l => l.name === finished.name);
+                                if (lane) {
+                                    lane.startIndex = state.currentTaskIndex;
+                                }
                             }
-                            catch (e) {
-                                logger.error(`[${laneName}] Failed to send guidance: ${e.message}`);
+                            // Note: we don't add to failedLanes or completedLanes, 
+                            // so it will be eligible to start again in the next iteration.
+                            continue;
+                        }
+                        failedLanes.add(finished.name);
+                        let errorMsg = 'Process exited with non-zero code';
+                        if (info.stallPhase === 3) {
+                            errorMsg = 'Stopped due to repeated stall';
+                        }
+                        else if (info.logManager) {
+                            const lastError = info.logManager.getLastError();
+                            if (lastError) {
+                                errorMsg = `Process failed: ${lastError}`;
                             }
                         }
+                        logger.error(`[${finished.name}] Lane failed with exit code ${finished.code}: ${errorMsg}`);
+                        // Log log tail for visibility
+                        if (info.logPath) {
+                            logFileTail(info.logPath, 15);
+                        }
+                        events_1.events.emit('lane.failed', {
+                            laneName: finished.name,
+                            exitCode: finished.code,
+                            error: errorMsg,
+                        });
                     }
-                }
-                continue;
-            }
-            const finished = result;
-            const info = running.get(finished.name);
-            running.delete(finished.name);
-            exitCodes[finished.name] = finished.code;
-            // Unregister from auto-recovery manager
-            autoRecoveryManager.unregisterLane(finished.name);
-            if (finished.code === 0) {
-                completedLanes.add(finished.name);
-                events_1.events.emit('lane.completed', {
-                    laneName: finished.name,
-                    exitCode: finished.code,
-                });
-            }
-            else if (finished.code === 2) {
-                // Blocked by dependency
-                const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
-                const state = (0, state_1.loadState)(statePath);
-                if (state && state.dependencyRequest) {
-                    blockedLanes.set(finished.name, state.dependencyRequest);
-                    const lane = lanes.find(l => l.name === finished.name);
-                    if (lane) {
-                        lane.startIndex = Math.max(0, state.currentTaskIndex - 1); // Task was blocked, retry it
-                    }
-                    events_1.events.emit('lane.blocked', {
-                        laneName: finished.name,
-                        dependencyRequest: state.dependencyRequest,
-                    });
-                    logger.warn(`Lane ${finished.name} is blocked on dependency change request`);
-                }
-                else {
-                    failedLanes.add(finished.name);
-                    logger.error(`Lane ${finished.name} exited with code 2 but no dependency request found`);
+                    printLaneStatus(lanes, laneRunDirs);
                 }
             }
             else {
-                // Check if it was a restart request
-                if (info.stallPhase === 2) {
-                    logger.info(`🔄 Lane ${finished.name} is being restarted due to stall...`);
-                    // Update startIndex from current state to resume from the same task
-                    const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
-                    const state = (0, state_1.loadState)(statePath);
-                    if (state) {
-                        const lane = lanes.find(l => l.name === finished.name);
-                        if (lane) {
-                            lane.startIndex = state.currentTaskIndex;
+                // Nothing running. Are we blocked?
+                // Wait a bit to avoid busy-spin while waiting for dependencies or new slots
+                if (completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length) {
+                    await new Promise(resolve => setTimeout(resolve, 2000));
+                }
+                if (blockedLanes.size > 0 && autoResolve) {
+                    logger.section('🛠 Auto-Resolving Dependencies');
+                    try {
+                        await resolveAllDependencies(blockedLanes, lanes, laneRunDirs, pipelineBranch, runRoot);
+                        // Clear blocked status
+                        blockedLanes.clear();
+                        logger.success('Dependencies resolved and synced across all active lanes. Resuming...');
+                    }
+                    catch (error) {
+                        logger.error(`Auto-resolution failed: ${error.message}`);
+                        // Move blocked to failed
+                        for (const name of blockedLanes.keys()) {
+                            failedLanes.add(name);
                         }
+                        blockedLanes.clear();
                     }
-                    // Note: we don't add to failedLanes or completedLanes, 
-                    // so it will be eligible to start again in the next iteration.
-                    continue;
-                }
-                failedLanes.add(finished.name);
-                events_1.events.emit('lane.failed', {
-                    laneName: finished.name,
-                    exitCode: finished.code,
-                    error: info.stallPhase === 3 ? 'Stopped due to repeated stall' : 'Process exited with non-zero code',
-                });
-            }
-            printLaneStatus(lanes, laneRunDirs);
-        }
-        else {
-            // Nothing running. Are we blocked?
-            if (blockedLanes.size > 0 && autoResolve) {
-                logger.section('🛠 Auto-Resolving Dependencies');
-                try {
-                    await resolveAllDependencies(blockedLanes, lanes, laneRunDirs, pipelineBranch, runRoot);
-                    // Clear blocked status
-                    blockedLanes.clear();
-                    logger.success('Dependencies resolved and synced across all active lanes. Resuming...');
                 }
-                catch (error) {
-                    logger.error(`Auto-resolution failed: ${error.message}`);
-                    // Move blocked to failed
-                    for (const name of blockedLanes.keys()) {
-                        failedLanes.add(name);
+                else if (readyToStart.length === 0 && completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length) {
+                    const remaining = lanes.filter(l => !completedLanes.has(l.name) && !failedLanes.has(l.name) && !blockedLanes.has(l.name));
+                    logger.error(`Deadlock detected! Remaining lanes cannot start: ${remaining.map(l => l.name).join(', ')}`);
+                    for (const l of remaining) {
+                        failedLanes.add(l.name);
+                        exitCodes[l.name] = 1;
                     }
-                    blockedLanes.clear();
                 }
-            }
-            else if (readyToStart.length === 0 && completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length) {
-                const remaining = lanes.filter(l => !completedLanes.has(l.name) && !failedLanes.has(l.name) && !blockedLanes.has(l.name));
-                logger.error(`Deadlock detected! Remaining lanes cannot start: ${remaining.map(l => l.name).join(', ')}`);
-                for (const l of remaining) {
-                    failedLanes.add(l.name);
-                    exitCodes[l.name] = 1;
+                else {
+                    // All finished
+                    break;
                 }
             }
-            else {
-                // All finished
-                break;
-            }
         }
     }
-    clearInterval(monitorInterval);
+    finally {
+        clearInterval(monitorInterval);
+        process.removeListener('SIGINT', sigIntHandler);
+        process.removeListener('SIGTERM', sigIntHandler);
+    }
     printLaneStatus(lanes, laneRunDirs);
     // Check for failures
     const failed = Object.entries(exitCodes).filter(([, code]) => code !== 0 && code !== 2);