@litmers/cursorflow-orchestrator 0.1.31 → 0.1.36

package/dist/core/orchestrator.js

@@ -60,15 +60,12 @@ const child_process_2 = require("child_process");
 const path_1 = require("../utils/path");
 const enhanced_logger_1 = require("../utils/enhanced-logger");
 const log_formatter_1 = require("../utils/log-formatter");
-const failure_policy_1 = require("./failure-policy");
 const auto_recovery_1 = require("./auto-recovery");
-const dependency_1 = require("../utils/dependency");
+const stall_detection_1 = require("./stall-detection");
 const health_1 = require("../utils/health");
-const checkpoint_1 = require("../utils/checkpoint");
 const lock_1 = require("../utils/lock");
 /** Default stall detection configuration - 2 minute idle timeout for recovery */
 const DEFAULT_ORCHESTRATOR_STALL_CONFIG = {
-    ...failure_policy_1.DEFAULT_STALL_CONFIG,
     idleTimeoutMs: 2 * 60 * 1000, // 2 minutes (idle detection for continue signal)
     progressTimeoutMs: 10 * 60 * 1000, // 10 minutes (only triggers if no activity at all)
     maxRestarts: 2,
@@ -94,10 +91,93 @@ function logFileTail(filePath, lines = 10) {
         // Ignore log reading errors
     }
 }
+/**
+ * Handle RUN_DOCTOR action - runs async health diagnostics
+ */
+async function handleDoctorDiagnostics(laneName, laneRunDir, runId, runRoot, stallService, child) {
+    // Import health check dynamically to avoid circular dependency
+    const { checkAgentHealth, checkAuthHealth } = await Promise.resolve().then(() => __importStar(require('../utils/health')));
+    const [agentHealth, authHealth] = await Promise.all([
+        checkAgentHealth(),
+        checkAuthHealth(),
+    ]);
+    const issues = [];
+    if (!agentHealth.ok)
+        issues.push(`Agent: ${agentHealth.message}`);
+    if (!authHealth.ok)
+        issues.push(`Auth: ${authHealth.message}`);
+    if (issues.length > 0) {
+        logger.error(`[${laneName}] Diagnostic issues found:\n  ${issues.join('\n  ')}`);
+    }
+    else {
+        logger.warn(`[${laneName}] No obvious issues found. The problem may be with the AI model or network.`);
+    }
+    // Save diagnostic to file
+    const diagnosticPath = (0, path_1.safeJoin)(laneRunDir, 'diagnostic.json');
+    fs.writeFileSync(diagnosticPath, JSON.stringify({
+        timestamp: Date.now(),
+        agentHealthy: agentHealth.ok,
+        authHealthy: authHealth.ok,
+        issues,
+    }, null, 2));
+    // Kill the process
+    try {
+        child.kill('SIGKILL');
+    }
+    catch {
+        // Process might already be dead
+    }
+    logger.error(`[${laneName}] Aborting lane after diagnostic. Check ${diagnosticPath} for details.`);
+    // Save POF for failed recovery
+    const stallState = stallService.getState(laneName);
+    if (stallState) {
+        try {
+            const laneStatePath = (0, path_1.safeJoin)(laneRunDir, 'state.json');
+            const laneState = (0, state_1.loadState)(laneStatePath);
+            const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
+            // Convert stall state to recovery state format for POF
+            // Note: StallPhase and RecoveryStage have compatible numeric values (0-5)
+            const recoveryState = {
+                laneName,
+                stage: stallState.phase, // Both enums use 0-5
+                lastActivityTime: stallState.lastRealActivityTime,
+                lastBytesReceived: stallState.bytesSinceLastCheck,
+                totalBytesReceived: stallState.totalBytesReceived,
+                lastOutput: stallState.lastOutput,
+                restartCount: stallState.restartCount,
+                continueSignalsSent: stallState.continueSignalCount,
+                lastStageChangeTime: stallState.lastPhaseChangeTime,
+                isLongOperation: stallState.isLongOperation,
+                failureHistory: stallState.failureHistory.map(f => ({
+                    timestamp: f.timestamp,
+                    stage: f.phase, // Both enums use 0-5
+                    action: f.action,
+                    message: f.message,
+                    idleTimeMs: f.idleTimeMs,
+                    bytesReceived: f.bytesReceived,
+                    lastOutput: f.lastOutput,
+                })),
+            };
+            const diagnosticInfo = {
+                timestamp: Date.now(),
+                agentHealthy: agentHealth.ok,
+                authHealthy: authHealth.ok,
+                systemHealthy: true,
+                suggestedAction: issues.length > 0 ? 'Fix the issues above and retry' : 'Try with a different model',
+                details: issues.join('\n') || 'No obvious issues found',
+            };
+            const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, diagnosticInfo);
+            (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
+        }
+        catch (pofError) {
+            logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
+        }
+    }
+}
 /**
  * Spawn a lane process
  */
-function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0, pipelineBranch, worktreeDir, enhancedLogConfig, noGit = false, onActivity, }) {
+function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0, pipelineBranch, worktreeDir, enhancedLogConfig, noGit = false, onActivity, laneIndex = 0, }) {
     fs.mkdirSync(laneRunDir, { recursive: true });
     // Use extension-less resolve to handle both .ts (dev) and .js (dist)
     const runnerPath = require.resolve('./runner');
@@ -127,17 +207,24 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
         ...process.env,
     };
     if (logConfig.enabled) {
+        // Helper to get dynamic lane label like [L1-T1-lanename10]
+        const getDynamicLabel = () => {
+            const laneNum = `L${laneIndex + 1}`;
+            const taskPart = info.currentTaskIndex ? `-T${info.currentTaskIndex}` : '';
+            const shortLaneName = laneName.substring(0, 10);
+            return `[${laneNum}${taskPart}-${shortLaneName}]`;
+        };
         // Create callback for clean console output
         const onParsedMessage = (msg) => {
             if (onActivity)
                 onActivity();
             const formatted = (0, log_formatter_1.formatMessageForConsole)(msg, {
-                laneLabel: `[${laneName}]`,
+                laneLabel: getDynamicLabel(),
                 includeTimestamp: true
             });
             process.stdout.write(formatted + '\n');
         };
-        logManager = (0, enhanced_logger_1.createLogManager)(laneRunDir, laneName, logConfig, onParsedMessage);
+        logManager = (0, enhanced_logger_1.createLogManager)(laneRunDir, laneName, logConfig, onParsedMessage, laneIndex);
         logPath = logManager.getLogPaths().clean;
         // Spawn with pipe for enhanced logging
         child = (0, child_process_1.spawn)('node', args, {
@@ -145,6 +232,15 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
             env: childEnv,
             detached: false,
         });
+        // Initialize info object for stdout handler to use
+        const info = {
+            child,
+            logManager,
+            logPath,
+            statePath: (0, path_1.safeJoin)(laneRunDir, 'state.json'),
+            laneIndex,
+            currentTaskIndex: startIndex > 0 ? startIndex + 1 : 0
+        };
         // Buffer for non-JSON lines
         let lineBuffer = '';
         // Pipe stdout and stderr through enhanced logger
@@ -158,25 +254,50 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
                 lineBuffer = lines.pop() || '';
                 for (const line of lines) {
                     const trimmed = line.trim();
+                    if (!trimmed)
+                        continue;
+                    // Detect task start/progress to update label
+                    // Example: [1/1] hello-task
+                    const cleanLine = (0, enhanced_logger_1.stripAnsi)(trimmed);
+                    const taskMatch = cleanLine.match(/^\s*\[(\d+)\/(\d+)\]\s+(.+)$/);
+                    if (taskMatch) {
+                        info.currentTaskIndex = parseInt(taskMatch[1]);
+                        // Update log manager's task index to keep it in sync for readable log
+                        if (logManager) {
+                            logManager.setTask(taskMatch[3].trim(), undefined, info.currentTaskIndex - 1);
+                        }
+                    }
                     // Show if it's a timestamped log line (starts with [YYYY-MM-DD... or [HH:MM:SS])
                     // or if it's NOT a noisy JSON line
-                    const hasTimestamp = /^\[\d{4}-\d{2}-\d{2}T|\^\[\d{2}:\d{2}:\d{2}\]/.test(trimmed);
                     const isJson = trimmed.startsWith('{') || trimmed.includes('{"type"');
                     // Filter out heartbeats - they should NOT reset the idle timer
                     const isHeartbeat = trimmed.includes('Heartbeat') && trimmed.includes('bytes received');
-                    if (trimmed && !isJson) {
+                    if (!isJson) {
                         // Only trigger activity for non-heartbeat lines
                         if (onActivity && !isHeartbeat)
                             onActivity();
-                        // If line alreedy has timestamp format, just add lane prefix
-                        if (hasTimestamp) {
-                            // Insert lane name after first timestamp
-                            const formatted = trimmed.replace(/^(\[[^\]]+\])/, `$1 ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset}`);
-                            process.stdout.write(formatted + '\n');
+                        const currentLabel = getDynamicLabel();
+                        const coloredLabel = `${logger.COLORS.magenta}${currentLabel}${logger.COLORS.reset}`;
+                        // Regex that matches timestamp even if it has ANSI color codes
+                        // Matches: [24:39:14] or \x1b[90m[24:39:14]\x1b[0m
+                        const timestampRegex = /^((?:\x1b\[[0-9;]*m)*)\[(\d{4}-\d{2}-\d{2}T|\d{2}:\d{2}:\d{2})\]/;
+                        const tsMatch = trimmed.match(timestampRegex);
+                        if (tsMatch) {
+                            // If line already has timestamp format, just add lane prefix
+                            // Check if lane label is already present to avoid triple duplication
+                            if (!trimmed.includes(currentLabel)) {
+                                // Insert label after the timestamp part
+                                const tsPart = tsMatch[0];
+                                const formatted = trimmed.replace(tsPart, `${tsPart} ${coloredLabel}`);
+                                process.stdout.write(formatted + '\n');
+                            }
+                            else {
+                                process.stdout.write(trimmed + '\n');
+                            }
                         }
                         else {
                             // Add full prefix: timestamp + lane
-                            process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${line}\n`);
+                            process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${coloredLabel} ${line}\n`);
                         }
                     }
                 }
@@ -196,13 +317,15 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
                             trimmed.startsWith('HEAD is now at') ||
                             trimmed.includes('actual output');
                         const ts = new Date().toLocaleTimeString('en-US', { hour12: false });
+                        const currentLabel = getDynamicLabel();
+                        const coloredLabel = `${logger.COLORS.magenta}${currentLabel}${logger.COLORS.reset}`;
                         if (isStatus) {
-                            process.stdout.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${trimmed}\n`);
+                            process.stdout.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${coloredLabel} ${trimmed}\n`);
                         }
                         else {
                             if (onActivity)
                                 onActivity();
-                            process.stderr.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${logger.COLORS.red}❌ ERR ${trimmed}${logger.COLORS.reset}\n`);
+                            process.stderr.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${coloredLabel} ${logger.COLORS.red}❌ ERR ${trimmed}${logger.COLORS.reset}\n`);
                         }
                     }
                 }
@@ -212,10 +335,11 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
         child.on('exit', () => {
             logManager?.close();
         });
+        return { child, logPath, logManager, info };
     }
     else {
         // Fallback to simple file logging
-        logPath = (0, path_1.safeJoin)(laneRunDir, 'terminal.log');
+        logPath = (0, path_1.safeJoin)(laneRunDir, 'terminal-readable.log');
         const logFd = fs.openSync(logPath, 'a');
         child = (0, child_process_1.spawn)('node', args, {
             stdio: ['ignore', logFd, logFd],
@@ -228,8 +352,18 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
         catch {
             // Ignore
         }
+        return {
+            child,
+            logPath,
+            logManager,
+            info: {
+                child,
+                logPath,
+                statePath: (0, path_1.safeJoin)(laneRunDir, 'state.json'),
+                laneIndex
+            }
+        };
     }
-    return { child, logPath, logManager };
 }
 /**
  * Wait for child process to exit
@@ -245,7 +379,7 @@ function waitChild(proc) {
     });
 }
 /**
- * List lane task files in directory and load their configs for dependencies
+ * List lane task files in directory
  */
 function listLaneFiles(tasksDir) {
     if (!fs.existsSync(tasksDir)) {
@@ -253,23 +387,14 @@ function listLaneFiles(tasksDir) {
     }
     const files = fs.readdirSync(tasksDir);
     return files
-        .filter(f => f.endsWith('.json'))
+        .filter(f => f.endsWith('.json') && f !== 'flow.meta.json')
         .sort()
         .map(f => {
         const filePath = (0, path_1.safeJoin)(tasksDir, f);
         const name = path.basename(f, '.json');
-        let dependsOn = [];
-        try {
-            const config = JSON.parse(fs.readFileSync(filePath, 'utf8'));
-            dependsOn = config.dependsOn || [];
-        }
-        catch (e) {
-            logger.warn(`Failed to parse config for lane ${name}: ${e}`);
-        }
         return {
             name,
             path: filePath,
-            dependsOn,
         };
     });
 }
@@ -284,8 +409,7 @@ function printLaneStatus(lanes, laneRunDirs) {
         const statePath = (0, path_1.safeJoin)(dir, 'state.json');
         const state = (0, state_1.loadState)(statePath);
         if (!state) {
-            const isWaiting = lane.dependsOn.length > 0;
-            return { lane: lane.name, status: isWaiting ? 'waiting' : 'pending', task: '-' };
+            return { lane: lane.name, status: 'pending', task: '-' };
         }
         const idx = (state.currentTaskIndex || 0) + 1;
         return {
@@ -322,11 +446,11 @@ async function resolveAllDependencies(blockedLanes, allLanes, laneRunDirs, pipel
     const state = (0, state_1.loadState)(statePath);
     const worktreeDir = state?.worktreeDir || (0, path_1.safeJoin)(runRoot, 'resolution-worktree');
     if (!fs.existsSync(worktreeDir)) {
-        logger.info(`Creating resolution worktree at ${worktreeDir}`);
+        logger.info(`🏗️ Creating resolution worktree at ${worktreeDir}`);
         git.createWorktree(worktreeDir, pipelineBranch, { baseBranch: git.getCurrentBranch() });
     }
     // 3. Resolve on pipeline branch
-    logger.info(`Resolving dependencies on ${pipelineBranch}`);
+    logger.info(`🔄 Resolving dependencies on branch ${pipelineBranch}`);
     git.runGit(['checkout', pipelineBranch], { cwd: worktreeDir });
     for (const cmd of uniqueCommands) {
         logger.info(`Running: ${cmd}`);
@@ -416,28 +540,9 @@ async function orchestrate(tasksDir, options = {}) {
         }
         logger.success('✓ Preflight checks passed');
     }
-    // Validate dependencies and detect cycles
-    logger.section('📊 Dependency Analysis');
-    const depInfos = lanes.map(l => ({
-        name: l.name,
-        dependsOn: l.dependsOn,
-    }));
-    const depValidation = (0, dependency_1.validateDependencies)(depInfos);
-    if (!depValidation.valid) {
-        logger.error('❌ Dependency validation failed:');
-        for (const err of depValidation.errors) {
-            logger.error(`   • ${err}`);
-        }
-        throw new Error('Invalid dependency configuration');
-    }
-    if (depValidation.warnings.length > 0) {
-        for (const warn of depValidation.warnings) {
-            logger.warn(`⚠️  ${warn}`);
-        }
-    }
-    // Print dependency graph
-    (0, dependency_1.printDependencyGraph)(depInfos);
     const config = (0, config_1.loadConfig)();
+    // Set verbose git logging from config
+    git.setVerboseGit(config.verboseGit || false);
     const logsDir = (0, config_1.getLogsDir)(config);
     const runId = `run-${Date.now()}`;
     // Use absolute path for runRoot to avoid issues with subfolders
@@ -458,16 +563,11 @@ async function orchestrate(tasksDir, options = {}) {
     }
     const randomSuffix = Math.random().toString(36).substring(2, 7);
     const pipelineBranch = `cursorflow/run-${Date.now().toString(36)}-${randomSuffix}`;
-    // Stall detection configuration
-    const stallConfig = {
+    // Initialize unified stall detection service (Single Source of Truth)
+    const stallService = (0, stall_detection_1.getStallService)({
         ...DEFAULT_ORCHESTRATOR_STALL_CONFIG,
         ...options.stallConfig,
-    };
-    // Initialize auto-recovery manager
-    const autoRecoveryManager = (0, auto_recovery_1.getAutoRecoveryManager)({
-        ...auto_recovery_1.DEFAULT_AUTO_RECOVERY_CONFIG,
-        idleTimeoutMs: stallConfig.idleTimeoutMs, // Sync with stall config
-        ...options.autoRecoveryConfig,
+        verbose: process.env['DEBUG_STALL'] === 'true',
     });
     // Initialize event system
     events_1.events.setRunId(runId);
@@ -512,6 +612,7 @@ async function orchestrate(tasksDir, options = {}) {
                 fs.mkdirSync(worktreeParent, { recursive: true });
             }
             laneWorktreeDirs[lane.name] = laneWorktreeDir;
+            logger.info(`🏗️ Initializing lane ${lane.name}: branch=${lanePipelineBranch}`);
             const initialState = (0, state_1.createLaneState)(lane.name, taskConfig, lane.path, {
                 pipelineBranch: lanePipelineBranch,
                 worktreeDir: laneWorktreeDir
@@ -526,19 +627,6 @@ async function orchestrate(tasksDir, options = {}) {
     logger.info(`Tasks directory: ${tasksDir}`);
     logger.info(`Run directory: ${runRoot}`);
     logger.info(`Lanes: ${lanes.length}`);
-    // Display dependency graph
-    logger.info('\n📊 Dependency Graph:');
-    for (const lane of lanes) {
-        const deps = lane.dependsOn.length > 0 ? ` [depends on: ${lane.dependsOn.join(', ')}]` : '';
-        console.log(`  ${logger.COLORS.cyan}${lane.name}${logger.COLORS.reset}${deps}`);
-        // Simple tree-like visualization for deep dependencies
-        if (lane.dependsOn.length > 0) {
-            for (const dep of lane.dependsOn) {
-                console.log(`    └─ ${dep}`);
-            }
-        }
-    }
-    console.log('');
     // Disable auto-resolve when noGit mode is enabled
     const autoResolve = !options.noGit && options.autoResolveDependencies !== false;
     if (options.noGit) {
@@ -568,28 +656,12 @@ async function orchestrate(tasksDir, options = {}) {
     let lastStallCheck = Date.now();
     try {
         while (completedLanes.size + failedLanes.size + blockedLanes.size < lanes.length || (blockedLanes.size > 0 && running.size === 0)) {
-            // 1. Identify lanes ready to start
+            // 1. Identify lanes ready to start (all lanes can start immediately - no lane-level dependencies)
             const readyToStart = lanes.filter(lane => {
                 // Not already running or completed or failed or blocked
                 if (running.has(lane.name) || completedLanes.has(lane.name) || failedLanes.has(lane.name) || blockedLanes.has(lane.name)) {
                     return false;
                 }
-                // Check dependencies
-                for (const dep of lane.dependsOn) {
-                    if (failedLanes.has(dep)) {
-                        logger.error(`Lane ${lane.name} will not start because dependency ${dep} failed`);
-                        failedLanes.add(lane.name);
-                        exitCodes[lane.name] = 1;
-                        return false;
-                    }
-                    if (blockedLanes.has(dep)) {
-                        // If a dependency is blocked, wait
-                        return false;
-                    }
-                    if (!completedLanes.has(dep)) {
-                        return false;
-                    }
-                }
                 return true;
             });
             // 2. Spawn ready lanes up to maxConcurrent
@@ -604,22 +676,19 @@ async function orchestrate(tasksDir, options = {}) {
                 }
                 logger.info(`Lane started: ${lane.name}${lane.startIndex ? ` (resuming from ${lane.startIndex})` : ''}`);
                 const now = Date.now();
-                // Pre-register lane in running map so onActivity can find it immediately
+                // Register lane with unified stall detection service FIRST
+                stallService.registerLane(lane.name, {
+                    laneRunDir: laneRunDirs[lane.name],
+                });
+                const laneIdx = lanes.findIndex(l => l.name === lane.name);
+                // Pre-register lane in running map
                 running.set(lane.name, {
                     child: {}, // Placeholder, will be replaced below
                     logManager: undefined,
                     logPath: '',
-                    lastActivity: now,
-                    lastStateUpdate: now,
-                    stallPhase: 0,
-                    taskStartTime: now,
-                    lastOutput: '',
                     statePath: laneStatePath,
-                    bytesReceived: 0,
-                    lastBytesCheck: 0,
-                    continueSignalsSent: 0,
+                    laneIndex: laneIdx >= 0 ? laneIdx : 0,
                 });
-                let lastOutput = '';
                 const spawnResult = spawnLane({
                     laneName: lane.name,
                     tasksFile: lane.path,
@@ -630,48 +699,35 @@ async function orchestrate(tasksDir, options = {}) {
                     worktreeDir: laneWorktreeDirs[lane.name],
                     enhancedLogConfig: options.enhancedLogging,
                     noGit: options.noGit,
+                    laneIndex: laneIdx >= 0 ? laneIdx : 0,
                     onActivity: () => {
-                        const info = running.get(lane.name);
-                        if (info) {
-                            const actNow = Date.now();
-                            info.lastActivity = actNow;
-                            info.lastStateUpdate = actNow;
-                            info.stallPhase = 0;
-                        }
+                        // Record state file update activity
+                        stallService.recordStateUpdate(lane.name);
                     }
                 });
                 // Update with actual spawn result
                 const existingInfo = running.get(lane.name);
-                Object.assign(existingInfo, spawnResult);
-                // Track last output and bytes received for long operation and stall detection
+                Object.assign(existingInfo, spawnResult.info);
+                // Update stall service with child process reference
+                stallService.setChildProcess(lane.name, spawnResult.child);
+                // Track stdout for activity detection - delegate to StallDetectionService
                 if (spawnResult.child.stdout) {
                     spawnResult.child.stdout.on('data', (data) => {
-                        const info = running.get(lane.name);
-                        if (info) {
-                            const output = data.toString();
-                            const lines = output.split('\n').filter(l => l.trim());
-                            // Filter out heartbeats from activity tracking to avoid resetting stall detection
-                            const realLines = lines.filter(line => !(line.includes('Heartbeat') && line.includes('bytes received')));
-                            if (realLines.length > 0) {
-                                // Real activity detected - update lastActivity to reset stall timer
-                                const actNow = Date.now();
-                                info.lastActivity = actNow;
-                                info.stallPhase = 0; // Reset stall phase on real activity
-                                const lastRealLine = realLines[realLines.length - 1];
-                                info.lastOutput = lastRealLine;
-                                info.bytesReceived += data.length;
-                                // Update auto-recovery manager with real activity
-                                autoRecoveryManager.recordActivity(lane.name, data.length, info.lastOutput);
-                            }
-                            else if (lines.length > 0) {
-                                // Only heartbeats received - do NOT update lastActivity (keep stall timer running)
-                                autoRecoveryManager.recordActivity(lane.name, 0, info.lastOutput);
-                            }
+                        const output = data.toString();
+                        const lines = output.split('\n').filter(l => l.trim());
+                        // Filter out heartbeats from activity tracking
+                        const realLines = lines.filter(line => !(line.includes('Heartbeat') && line.includes('bytes received')));
+                        if (realLines.length > 0) {
+                            // Real activity - record with bytes
+                            const lastRealLine = realLines[realLines.length - 1];
+                            stallService.recordActivity(lane.name, data.length, lastRealLine);
+                        }
+                        else if (lines.length > 0) {
+                            // Heartbeat only - record with 0 bytes (won't reset timer)
+                            stallService.recordActivity(lane.name, 0);
                         }
                     });
                 }
-                // Register lane with auto-recovery manager
-                autoRecoveryManager.registerLane(lane.name);
                 // Update lane tracking
                 lane.taskStartTime = now;
                 events_1.events.emit('lane.started', {
@@ -697,212 +753,35 @@ async function orchestrate(tasksDir, options = {}) {
                 const now = Date.now();
                 if (result.name === '__poll__' || (now - lastStallCheck >= 10000)) {
                     lastStallCheck = now;
-                    // Periodic stall check with multi-layer detection and escalating recovery
+                    // Periodic stall check using unified StallDetectionService
                     for (const [laneName, info] of running.entries()) {
-                        const idleTime = now - info.lastActivity;
                         const lane = lanes.find(l => l.name === laneName);
-                        if (process.env['DEBUG_STALL']) {
-                            logger.debug(`[${laneName}] Stall check: idle=${Math.round(idleTime / 1000)}s, bytesDelta=${info.bytesReceived - info.lastBytesCheck}, phase=${info.stallPhase}`);
-                        }
                         // Check state file for progress updates
-                        let progressTime = 0;
                         try {
                             const stateStat = fs.statSync(info.statePath);
-                            const stateUpdateTime = stateStat.mtimeMs;
-                            if (stateUpdateTime > info.lastStateUpdate) {
-                                info.lastStateUpdate = stateUpdateTime;
+                            const stallState = stallService.getState(laneName);
+                            if (stallState && stateStat.mtimeMs > stallState.lastStateUpdateTime) {
+                                stallService.recordStateUpdate(laneName);
                             }
-                            progressTime = now - info.lastStateUpdate;
                         }
                         catch {
                             // State file might not exist yet
                         }
-                        // Calculate bytes received since last check
-                        const bytesDelta = info.bytesReceived - info.lastBytesCheck;
-                        info.lastBytesCheck = info.bytesReceived;
-                        // Use multi-layer stall analysis with enhanced context
-                        const analysis = (0, failure_policy_1.analyzeStall)({
-                            stallPhase: info.stallPhase,
-                            idleTimeMs: idleTime,
-                            progressTimeMs: progressTime,
-                            lastOutput: info.lastOutput,
-                            restartCount: lane.restartCount || 0,
-                            taskStartTimeMs: info.taskStartTime,
-                            bytesReceived: bytesDelta, // Bytes since last check
-                            continueSignalsSent: info.continueSignalsSent,
-                        }, stallConfig);
-                        // Only act if action is not NONE
-                        if (analysis.action !== failure_policy_1.RecoveryAction.NONE) {
-                            (0, failure_policy_1.logFailure)(laneName, analysis);
+                        // Debug logging
+                        if (process.env['DEBUG_STALL']) {
+                            logger.debug(`[${laneName}] ${stallService.dumpState(laneName)}`);
+                        }
+                        // Run stall analysis and recovery (all logic is in StallDetectionService)
+                        const analysis = stallService.checkAndRecover(laneName);
+                        // Log to lane log manager if there was an action
+                        if (analysis.action !== stall_detection_1.RecoveryAction.NONE) {
                             info.logManager?.log('error', analysis.message);
-                            if (analysis.action === failure_policy_1.RecoveryAction.CONTINUE_SIGNAL) {
-                                const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
-                                try {
-                                    fs.writeFileSync(interventionPath, 'continue');
-                                    info.stallPhase = 1;
-                                    info.lastActivity = now;
-                                    info.continueSignalsSent++;
-                                    logger.info(`[${laneName}] Sent continue signal (#${info.continueSignalsSent})`);
-                                    events_1.events.emit('recovery.continue_signal', {
-                                        laneName,
-                                        idleSeconds: Math.round(idleTime / 1000),
-                                        signalCount: info.continueSignalsSent,
-                                    });
-                                }
-                                catch (e) {
-                                    logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
-                                }
-                            }
-                            else if (analysis.action === failure_policy_1.RecoveryAction.STRONGER_PROMPT) {
-                                const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
-                                const strongerPrompt = `[SYSTEM INTERVENTION] You seem to be stuck. Please continue with your current task immediately. If you're waiting for something, explain what you need and proceed with what you can do now. If you've completed the task, summarize your work and finish.`;
-                                try {
-                                    fs.writeFileSync(interventionPath, strongerPrompt);
-                                    info.stallPhase = 2;
-                                    info.lastActivity = now;
-                                    logger.warn(`[${laneName}] Sent stronger prompt after continue signal failed`);
-                                    events_1.events.emit('recovery.stronger_prompt', { laneName });
-                                }
-                                catch (e) {
-                                    logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
-                                }
-                            }
-                            else if (analysis.action === failure_policy_1.RecoveryAction.KILL_AND_RESTART ||
-                                analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE ||
-                                analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE_FROM_CHECKPOINT) {
-                                lane.restartCount = (lane.restartCount || 0) + 1;
-                                info.stallPhase = 3;
-                                // Try to get checkpoint info
-                                const checkpoint = (0, checkpoint_1.getLatestCheckpoint)(laneRunDirs[laneName]);
-                                if (checkpoint) {
-                                    logger.info(`[${laneName}] Checkpoint available: ${checkpoint.id} (task ${checkpoint.taskIndex})`);
-                                }
-                                // Kill the process
-                                try {
-                                    info.child.kill('SIGKILL');
-                                }
-                                catch {
-                                    // Process might already be dead
-                                }
-                                logger.warn(`[${laneName}] Killing and restarting lane (restart #${lane.restartCount})`);
-                                events_1.events.emit('recovery.restart', {
-                                    laneName,
-                                    restartCount: lane.restartCount,
-                                    maxRestarts: stallConfig.maxRestarts,
-                                });
-                            }
-                            else if (analysis.action === failure_policy_1.RecoveryAction.RUN_DOCTOR) {
-                                info.stallPhase = 4;
-                                // Run diagnostics
-                                logger.error(`[${laneName}] Running diagnostics due to persistent failures...`);
-                                // Import health check dynamically to avoid circular dependency
-                                const { checkAgentHealth, checkAuthHealth } = await Promise.resolve().then(() => __importStar(require('../utils/health')));
-                                const [agentHealth, authHealth] = await Promise.all([
-                                    checkAgentHealth(),
-                                    checkAuthHealth(),
-                                ]);
-                                const issues = [];
-                                if (!agentHealth.ok)
-                                    issues.push(`Agent: ${agentHealth.message}`);
-                                if (!authHealth.ok)
-                                    issues.push(`Auth: ${authHealth.message}`);
-                                if (issues.length > 0) {
-                                    logger.error(`[${laneName}] Diagnostic issues found:\n  ${issues.join('\n  ')}`);
-                                }
-                                else {
-                                    logger.warn(`[${laneName}] No obvious issues found. The problem may be with the AI model or network.`);
-                                }
-                                // Save diagnostic to file
-                                const diagnosticPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'diagnostic.json');
-                                fs.writeFileSync(diagnosticPath, JSON.stringify({
-                                    timestamp: Date.now(),
-                                    agentHealthy: agentHealth.ok,
-                                    authHealthy: authHealth.ok,
-                                    issues,
-                                    analysis,
-                                }, null, 2));
-                                // Kill the process
-                                try {
-                                    info.child.kill('SIGKILL');
-                                }
-                                catch {
-                                    // Process might already be dead
-                                }
-                                logger.error(`[${laneName}] Aborting lane after diagnostic. Check ${diagnosticPath} for details.`);
-                                // Save POF for failed recovery
-                                const recoveryState = autoRecoveryManager.getState(laneName);
-                                if (recoveryState) {
-                                    try {
-                                        const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
-                                        const laneState = (0, state_1.loadState)(laneStatePath);
-                                        const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
-                                        const diagnosticInfo = {
-                                            timestamp: Date.now(),
-                                            agentHealthy: agentHealth.ok,
-                                            authHealthy: authHealth.ok,
-                                            systemHealthy: true,
-                                            suggestedAction: issues.length > 0 ? 'Fix the issues above and retry' : 'Try with a different model',
-                                            details: issues.join('\n') || 'No obvious issues found',
-                                        };
-                                        const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, diagnosticInfo);
-                                        (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
-                                    }
-                                    catch (pofError) {
-                                        logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
-                                    }
-                                }
-                                events_1.events.emit('recovery.diagnosed', {
-                                    laneName,
-                                    diagnostic: { agentHealthy: agentHealth.ok, authHealthy: authHealth.ok, issues },
-                                });
-                            }
-                            else if (analysis.action === failure_policy_1.RecoveryAction.ABORT_LANE) {
-                                info.stallPhase = 5;
-                                try {
-                                    info.child.kill('SIGKILL');
-                                }
-                                catch {
-                                    // Process might already be dead
-                                }
-                                logger.error(`[${laneName}] Aborting lane due to repeated stalls`);
-                                // Save POF for failed recovery
-                                const recoveryState = autoRecoveryManager.getState(laneName);
-                                if (recoveryState) {
-                                    try {
-                                        const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
-                                        const laneState = (0, state_1.loadState)(laneStatePath);
-                                        const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
-                                        const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, recoveryState.diagnosticInfo);
-                                        (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
-                                    }
-                                    catch (pofError) {
-                                        logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
-                                    }
-                                }
-                            }
-                            else if (analysis.action === failure_policy_1.RecoveryAction.SEND_GIT_GUIDANCE) {
-                                // Send guidance message to agent for git issues
-                                const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
-                                // Determine which guidance to send based on the failure type
-                                let guidance;
-                                if (analysis.type === failure_policy_1.FailureType.GIT_PUSH_REJECTED) {
-                                    guidance = (0, auto_recovery_1.getGitPushFailureGuidance)();
-                                }
-                                else if (analysis.type === failure_policy_1.FailureType.MERGE_CONFLICT) {
-                                    guidance = (0, auto_recovery_1.getMergeConflictGuidance)();
-                                }
-                                else {
-                                    guidance = (0, auto_recovery_1.getGitErrorGuidance)(analysis.message);
-                                }
-                                try {
-                                    fs.writeFileSync(interventionPath, guidance);
-                                    info.lastActivity = now;
-                                    logger.info(`[${laneName}] Sent git issue guidance to agent`);
-                                }
-                                catch (e) {
-                                    logger.error(`[${laneName}] Failed to send guidance: ${e.message}`);
-                                }
+                            // Handle special case: RUN_DOCTOR needs async operations
+                            if (analysis.action === stall_detection_1.RecoveryAction.RUN_DOCTOR) {
+                                await handleDoctorDiagnostics(laneName, laneRunDirs[laneName], runId, runRoot, stallService, info.child);
                             }
+                            // Sync restartCount back to lane info (for restart logic in process exit handler)
+                            lane.restartCount = stallService.getRestartCount(laneName);
                         }
                     }
                     continue;
@@ -912,8 +791,10 @@ async function orchestrate(tasksDir, options = {}) {
                     const info = running.get(finished.name);
                     running.delete(finished.name);
                     exitCodes[finished.name] = finished.code;
-                    // Unregister from auto-recovery manager
-                    autoRecoveryManager.unregisterLane(finished.name);
+                    // Get stall state before unregistering
+                    const stallPhase = stallService.getPhase(finished.name);
+                    // Unregister from stall detection service
+                    stallService.unregisterLane(finished.name);
                     if (finished.code === 0) {
                         completedLanes.add(finished.name);
                         events_1.events.emit('lane.completed', {
@@ -943,8 +824,8 @@ async function orchestrate(tasksDir, options = {}) {
                         }
                     }
                     else {
-                        // Check if it was a restart request
-                        if (info.stallPhase === 2) {
+                        // Check if it was a restart request (RESTART_REQUESTED phase)
+                        if (stallPhase === stall_detection_1.StallPhase.RESTART_REQUESTED) {
                             logger.info(`🔄 Lane ${finished.name} is being restarted due to stall...`);
                             // Update startIndex from current state to resume from the same task
                             const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
@@ -961,7 +842,7 @@ async function orchestrate(tasksDir, options = {}) {
                         }
                         failedLanes.add(finished.name);
                         let errorMsg = 'Process exited with non-zero code';
-                        if (info.stallPhase === 3) {
+                        if (stallPhase >= stall_detection_1.StallPhase.DIAGNOSED) {
                             errorMsg = 'Stopped due to repeated stall';
                         }
                         else if (info.logManager) {