@litmers/cursorflow-orchestrator 0.1.18 → 0.1.26

package/dist/core/orchestrator.js

@@ -2,7 +2,11 @@
 /**
  * Orchestrator - Parallel lane execution with dependency management
  *
- * Adapted from admin-domains-orchestrator.js
+ * Features:
+ * - Multi-layer stall detection
+ * - Cyclic dependency detection
+ * - Enhanced recovery strategies
+ * - Health checks before start
  */
 var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
     if (k2 === undefined) k2 = k;
@@ -53,11 +57,26 @@ const webhook_1 = require("../utils/webhook");
 const config_1 = require("../utils/config");
 const git = __importStar(require("../utils/git"));
 const child_process_2 = require("child_process");
+const path_1 = require("../utils/path");
 const enhanced_logger_1 = require("../utils/enhanced-logger");
+const log_formatter_1 = require("../utils/log-formatter");
+const failure_policy_1 = require("./failure-policy");
+const auto_recovery_1 = require("./auto-recovery");
+const dependency_1 = require("../utils/dependency");
+const health_1 = require("../utils/health");
+const checkpoint_1 = require("../utils/checkpoint");
+const lock_1 = require("../utils/lock");
+/** Default stall detection configuration - 1 minute idle timeout for fast recovery */
+const DEFAULT_ORCHESTRATOR_STALL_CONFIG = {
+    ...failure_policy_1.DEFAULT_STALL_CONFIG,
+    idleTimeoutMs: 60 * 1000, // 1 minute (quick detection for continue signal)
+    progressTimeoutMs: 10 * 60 * 1000, // 10 minutes
+    maxRestarts: 2,
+};
 /**
  * Spawn a lane process
  */
-function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0, pipelineBranch, enhancedLogConfig, noGit = false, }) {
+function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0, pipelineBranch, worktreeDir, enhancedLogConfig, noGit = false, onActivity, }) {
     fs.mkdirSync(laneRunDir, { recursive: true });
     // Use extension-less resolve to handle both .ts (dev) and .js (dist)
     const runnerPath = require.resolve('./runner');
@@ -71,6 +90,9 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
     if (pipelineBranch) {
         args.push('--pipeline-branch', pipelineBranch);
     }
+    if (worktreeDir) {
+        args.push('--worktree-dir', worktreeDir);
+    }
     if (noGit) {
         args.push('--no-git');
     }
@@ -86,86 +108,13 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
     if (logConfig.enabled) {
         // Create callback for clean console output
         const onParsedMessage = (msg) => {
-            // Print a clean, colored version of the message to the console
-            const ts = new Date(msg.timestamp).toLocaleTimeString('en-US', { hour12: false });
-            const laneLabel = `[${laneName}]`.padEnd(12);
-            let prefix = '';
-            let content = msg.content;
-            switch (msg.type) {
-                case 'user':
-                    prefix = `${logger.COLORS.cyan}🧑 USER${logger.COLORS.reset}`;
-                    // No truncation for user prompt to ensure full command visibility
-                    content = content.replace(/\n/g, ' ');
-                    break;
-                case 'assistant':
-                    prefix = `${logger.COLORS.green}🤖 ASST${logger.COLORS.reset}`;
-                    break;
-                case 'tool':
-                    prefix = `${logger.COLORS.yellow}🔧 TOOL${logger.COLORS.reset}`;
-                    // Simplify tool call: [Tool: read_file] {"target_file":"..."} -> read_file(target_file: ...)
-                    const toolMatch = content.match(/\[Tool: ([^\]]+)\] (.*)/);
-                    if (toolMatch) {
-                        const [, name, args] = toolMatch;
-                        try {
-                            const parsedArgs = JSON.parse(args);
-                            let argStr = '';
-                            if (name === 'read_file' && parsedArgs.target_file) {
-                                argStr = parsedArgs.target_file;
-                            }
-                            else if (name === 'run_terminal_cmd' && parsedArgs.command) {
-                                argStr = parsedArgs.command;
-                            }
-                            else if (name === 'write' && parsedArgs.file_path) {
-                                argStr = parsedArgs.file_path;
-                            }
-                            else if (name === 'search_replace' && parsedArgs.file_path) {
-                                argStr = parsedArgs.file_path;
-                            }
-                            else {
-                                // Generic summary for other tools
-                                const keys = Object.keys(parsedArgs);
-                                if (keys.length > 0) {
-                                    argStr = String(parsedArgs[keys[0]]).substring(0, 50);
-                                }
-                            }
-                            content = `${logger.COLORS.bold}${name}${logger.COLORS.reset}(${argStr})`;
-                        }
-                        catch {
-                            content = `${logger.COLORS.bold}${name}${logger.COLORS.reset}: ${args}`;
-                        }
-                    }
-                    break;
-                case 'tool_result':
-                    prefix = `${logger.COLORS.gray}📄 RESL${logger.COLORS.reset}`;
-                    // Simplify tool result: [Tool Result: read_file] ... -> read_file OK
-                    const resMatch = content.match(/\[Tool Result: ([^\]]+)\]/);
-                    content = resMatch ? `${resMatch[1]} OK` : 'result';
-                    break;
-                case 'result':
-                    prefix = `${logger.COLORS.green}✅ DONE${logger.COLORS.reset}`;
-                    break;
-                case 'system':
-                    prefix = `${logger.COLORS.gray}⚙️  SYS${logger.COLORS.reset}`;
-                    break;
-                case 'thinking':
-                    prefix = `${logger.COLORS.gray}🤔 THNK${logger.COLORS.reset}`;
-                    break;
-            }
-            if (prefix) {
-                const lines = content.split('\n');
-                const tsPrefix = `${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${logger.COLORS.magenta}${laneLabel}${logger.COLORS.reset}`;
-                if (msg.type === 'user' || msg.type === 'assistant' || msg.type === 'result' || msg.type === 'thinking') {
-                    const header = `${prefix} ┌${'─'.repeat(60)}`;
-                    process.stdout.write(`${tsPrefix} ${header}\n`);
-                    for (const line of lines) {
-                        process.stdout.write(`${tsPrefix} ${' '.repeat((0, enhanced_logger_1.stripAnsi)(prefix).length)} │ ${line}\n`);
-                    }
-                    process.stdout.write(`${tsPrefix} ${' '.repeat((0, enhanced_logger_1.stripAnsi)(prefix).length)} └${'─'.repeat(60)}\n`);
-                }
-                else {
-                    process.stdout.write(`${tsPrefix} ${prefix} ${content}\n`);
-                }
-            }
+            if (onActivity)
+                onActivity();
+            const formatted = (0, log_formatter_1.formatMessageForConsole)(msg, {
+                laneLabel: `[${laneName}]`,
+                includeTimestamp: true
+            });
+            process.stdout.write(formatted + '\n');
         };
         logManager = (0, enhanced_logger_1.createLogManager)(laneRunDir, laneName, logConfig, onParsedMessage);
         logPath = logManager.getLogPaths().clean;
@@ -188,12 +137,23 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
                 lineBuffer = lines.pop() || '';
                 for (const line of lines) {
                     const trimmed = line.trim();
-                    // Only print if NOT a noisy line
-                    if (trimmed &&
-                        !trimmed.startsWith('{') &&
-                        !trimmed.startsWith('[') &&
-                        !trimmed.includes('{"type"')) {
-                        process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${logger.COLORS.magenta}${laneName.padEnd(10)}${logger.COLORS.reset} ${line}\n`);
+                    // Show if it's a timestamped log line (starts with [YYYY-MM-DD... or [HH:MM:SS])
+                    // or if it's NOT a noisy JSON line
+                    const hasTimestamp = /^\[\d{4}-\d{2}-\d{2}T|\^\[\d{2}:\d{2}:\d{2}\]/.test(trimmed);
+                    const isJson = trimmed.startsWith('{') || trimmed.includes('{"type"');
+                    if (trimmed && !isJson) {
+                        if (onActivity)
+                            onActivity();
+                        // If line already has timestamp format, just add lane prefix
+                        if (hasTimestamp) {
+                            // Insert lane name after first timestamp
+                            const formatted = trimmed.replace(/^(\[[^\]]+\])/, `$1 ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset}`);
+                            process.stdout.write(formatted + '\n');
+                        }
+                        else {
+                            // Add full prefix: timestamp + lane
+                            process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${line}\n`);
+                        }
                     }
                 }
             });
@@ -211,11 +171,14 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
                             trimmed.startsWith('Switched to a new branch') ||
                             trimmed.startsWith('HEAD is now at') ||
                             trimmed.includes('actual output');
+                        const ts = new Date().toLocaleTimeString('en-US', { hour12: false });
                         if (isStatus) {
-                            process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${logger.COLORS.magenta}${laneName.padEnd(10)}${logger.COLORS.reset} ${trimmed}\n`);
+                            process.stdout.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${trimmed}\n`);
                         }
                         else {
-                            process.stderr.write(`${logger.COLORS.red}[${laneName}] ERROR: ${trimmed}${logger.COLORS.reset}\n`);
+                            if (onActivity)
+                                onActivity();
+                            process.stderr.write(`${logger.COLORS.gray}[${ts}]${logger.COLORS.reset} ${logger.COLORS.magenta}[${laneName}]${logger.COLORS.reset} ${logger.COLORS.red}❌ ERR ${trimmed}${logger.COLORS.reset}\n`);
                         }
                     }
                 }
@@ -228,7 +191,7 @@ function spawnLane({ laneName, tasksFile, laneRunDir, executor, startIndex = 0,
     }
     else {
         // Fallback to simple file logging
-        logPath = path.join(laneRunDir, 'terminal.log');
+        logPath = (0, path_1.safeJoin)(laneRunDir, 'terminal.log');
         const logFd = fs.openSync(logPath, 'a');
         child = (0, child_process_1.spawn)('node', args, {
             stdio: ['ignore', logFd, logFd],
@@ -269,7 +232,7 @@ function listLaneFiles(tasksDir) {
         .filter(f => f.endsWith('.json'))
         .sort()
         .map(f => {
-        const filePath = path.join(tasksDir, f);
+        const filePath = (0, path_1.safeJoin)(tasksDir, f);
         const name = path.basename(f, '.json');
         let dependsOn = [];
         try {
@@ -294,7 +257,7 @@ function printLaneStatus(lanes, laneRunDirs) {
         const dir = laneRunDirs[lane.name];
         if (!dir)
             return { lane: lane.name, status: '(unknown)', task: '-' };
-        const statePath = path.join(dir, 'state.json');
+        const statePath = (0, path_1.safeJoin)(dir, 'state.json');
         const state = (0, state_1.loadState)(statePath);
         if (!state) {
             const isWaiting = lane.dependsOn.length > 0;
@@ -331,12 +294,12 @@ async function resolveAllDependencies(blockedLanes, allLanes, laneRunDirs, pipel
         return;
     // 2. Setup a temporary worktree for resolution if needed, or use the first available one
     const firstLaneName = Array.from(blockedLanes.keys())[0];
-    const statePath = path.join(laneRunDirs[firstLaneName], 'state.json');
+    const statePath = (0, path_1.safeJoin)(laneRunDirs[firstLaneName], 'state.json');
     const state = (0, state_1.loadState)(statePath);
-    const worktreeDir = state?.worktreeDir || path.join(runRoot, 'resolution-worktree');
+    const worktreeDir = state?.worktreeDir || (0, path_1.safeJoin)(runRoot, 'resolution-worktree');
     if (!fs.existsSync(worktreeDir)) {
         logger.info(`Creating resolution worktree at ${worktreeDir}`);
-        git.createWorktree(worktreeDir, pipelineBranch, { baseBranch: 'main' });
+        git.createWorktree(worktreeDir, pipelineBranch, { baseBranch: git.getCurrentBranch() });
     }
     // 3. Resolve on pipeline branch
     logger.info(`Resolving dependencies on ${pipelineBranch}`);
@@ -367,7 +330,7 @@ async function resolveAllDependencies(blockedLanes, allLanes, laneRunDirs, pipel
         const laneDir = laneRunDirs[lane.name];
         if (!laneDir)
             continue;
-        const laneState = (0, state_1.loadState)(path.join(laneDir, 'state.json'));
+        const laneState = (0, state_1.loadState)((0, path_1.safeJoin)(laneDir, 'state.json'));
         if (!laneState || laneState.status === 'completed' || laneState.status === 'failed')
             continue;
         // Merge pipelineBranch into the lane's current task branch
@@ -406,16 +369,82 @@ async function orchestrate(tasksDir, options = {}) {
     if (lanes.length === 0) {
         throw new Error(`No lane task files found in ${tasksDir}`);
     }
+    // Run preflight checks
+    if (!options.skipPreflight) {
+        logger.section('🔍 Preflight Checks');
+        const preflight = await (0, health_1.preflightCheck)({
+            requireRemote: !options.noGit,
+            requireAuth: true,
+        });
+        if (!preflight.canProceed) {
+            (0, health_1.printPreflightReport)(preflight);
+            throw new Error('Preflight check failed. Please fix the blockers above.');
+        }
+        // Auto-repair if there are warnings
+        if (preflight.warnings.length > 0) {
+            logger.info('Attempting auto-repair...');
+            const repair = await (0, health_1.autoRepair)();
+            if (repair.repaired.length > 0) {
+                for (const r of repair.repaired) {
+                    logger.success(`✓ ${r}`);
+                }
+            }
+        }
+        logger.success('✓ Preflight checks passed');
+    }
+    // Validate dependencies and detect cycles
+    logger.section('📊 Dependency Analysis');
+    const depInfos = lanes.map(l => ({
+        name: l.name,
+        dependsOn: l.dependsOn,
+    }));
+    const depValidation = (0, dependency_1.validateDependencies)(depInfos);
+    if (!depValidation.valid) {
+        logger.error('❌ Dependency validation failed:');
+        for (const err of depValidation.errors) {
+            logger.error(`   • ${err}`);
+        }
+        throw new Error('Invalid dependency configuration');
+    }
+    if (depValidation.warnings.length > 0) {
+        for (const warn of depValidation.warnings) {
+            logger.warn(`⚠️  ${warn}`);
+        }
+    }
+    // Print dependency graph
+    (0, dependency_1.printDependencyGraph)(depInfos);
     const config = (0, config_1.loadConfig)();
     const logsDir = (0, config_1.getLogsDir)(config);
     const runId = `run-${Date.now()}`;
     // Use absolute path for runRoot to avoid issues with subfolders
     const runRoot = options.runDir
-        ? (path.isAbsolute(options.runDir) ? options.runDir : path.resolve(process.cwd(), options.runDir))
-        : path.join(logsDir, 'runs', runId);
+        ? (path.isAbsolute(options.runDir) ? options.runDir : path.resolve(process.cwd(), options.runDir)) // nosemgrep
+        : (0, path_1.safeJoin)(logsDir, 'runs', runId);
     fs.mkdirSync(runRoot, { recursive: true });
+    // Clean stale locks before starting
+    try {
+        const lockDir = (0, lock_1.getLockDir)(git.getRepoRoot());
+        const cleaned = (0, lock_1.cleanStaleLocks)(lockDir);
+        if (cleaned > 0) {
+            logger.info(`Cleaned ${cleaned} stale lock(s)`);
+        }
+    }
+    catch {
+        // Ignore lock cleanup errors
+    }
     const randomSuffix = Math.random().toString(36).substring(2, 7);
     const pipelineBranch = `cursorflow/run-${Date.now().toString(36)}-${randomSuffix}`;
+    // Stall detection configuration
+    const stallConfig = {
+        ...DEFAULT_ORCHESTRATOR_STALL_CONFIG,
+        ...options.stallConfig,
+    };
+    // Initialize auto-recovery manager
+    const autoRecoveryManager = (0, auto_recovery_1.getAutoRecoveryManager)({
+        ...auto_recovery_1.DEFAULT_AUTO_RECOVERY_CONFIG,
+        idleTimeoutMs: stallConfig.idleTimeoutMs, // Sync with stall config
+        ...options.autoRecoveryConfig,
+    });
     // Initialize event system
     events_1.events.setRunId(runId);
     if (options.webhooks) {
@@ -436,11 +465,38 @@ async function orchestrate(tasksDir, options = {}) {
     // Track start index for each lane (initially 0)
     for (const lane of lanes) {
         lane.startIndex = 0;
+        lane.restartCount = 0;
     }
     const laneRunDirs = {};
+    const laneWorktreeDirs = {};
+    const repoRoot = git.getRepoRoot();
     for (const lane of lanes) {
-        laneRunDirs[lane.name] = path.join(runRoot, 'lanes', lane.name);
+        laneRunDirs[lane.name] = (0, path_1.safeJoin)(runRoot, 'lanes', lane.name);
         fs.mkdirSync(laneRunDirs[lane.name], { recursive: true });
+        // Create initial state for ALL lanes so resume can find them even if they didn't start
+        try {
+            const taskConfig = JSON.parse(fs.readFileSync(lane.path, 'utf8'));
+            // Calculate unique branch and worktree for this lane
+            const lanePipelineBranch = `${pipelineBranch}/${lane.name}`;
+            // Use a flat worktree directory name to avoid race conditions in parent directory creation
+            // repoRoot/_cursorflow/worktrees/cursorflow-run-xxx-lane-name
+            const laneWorktreeDir = (0, path_1.safeJoin)(repoRoot, taskConfig.worktreeRoot || '_cursorflow/worktrees', lanePipelineBranch.replace(/\//g, '-'));
+            // Ensure the parent directory exists before spawning the runner
+            // to avoid race conditions in git worktree add or fs operations
+            const worktreeParent = path.dirname(laneWorktreeDir);
+            if (!fs.existsSync(worktreeParent)) {
+                fs.mkdirSync(worktreeParent, { recursive: true });
+            }
+            laneWorktreeDirs[lane.name] = laneWorktreeDir;
+            const initialState = (0, state_1.createLaneState)(lane.name, taskConfig, lane.path, {
+                pipelineBranch: lanePipelineBranch,
+                worktreeDir: laneWorktreeDir
+            });
+            (0, state_1.saveState)((0, path_1.safeJoin)(laneRunDirs[lane.name], 'state.json'), initialState);
+        }
+        catch (e) {
+            logger.warn(`Failed to create initial state for lane ${lane.name}: ${e}`);
+        }
     }
     logger.section('🧭 Starting Orchestration');
     logger.info(`Tasks directory: ${tasksDir}`);
@@ -497,7 +553,14 @@ async function orchestrate(tasksDir, options = {}) {
         for (const lane of readyToStart) {
             if (running.size >= maxConcurrent)
                 break;
+            const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[lane.name], 'state.json');
+            // Validate and repair state before starting
+            const validation = (0, state_1.validateLaneState)(laneStatePath, { autoRepair: true });
+            if (!validation.valid && !validation.repaired) {
+                logger.warn(`[${lane.name}] State validation issues: ${validation.issues.join(', ')}`);
+            }
             logger.info(`Lane started: ${lane.name}${lane.startIndex ? ` (resuming from ${lane.startIndex})` : ''}`);
+            let lastOutput = '';
             const spawnResult = spawnLane({
                 laneName: lane.name,
                 tasksFile: lane.path,
@@ -505,25 +568,280 @@ async function orchestrate(tasksDir, options = {}) {
                 executor: options.executor || 'cursor-agent',
                 startIndex: lane.startIndex,
                 pipelineBranch: `${pipelineBranch}/${lane.name}`,
+                worktreeDir: laneWorktreeDirs[lane.name],
                 enhancedLogConfig: options.enhancedLogging,
                 noGit: options.noGit,
+                onActivity: () => {
+                    const info = running.get(lane.name);
+                    if (info) {
+                        info.lastActivity = Date.now();
+                    }
+                }
             });
-            running.set(lane.name, spawnResult);
+            // Track last output and bytes received for long operation and stall detection
+            if (spawnResult.child.stdout) {
+                spawnResult.child.stdout.on('data', (data) => {
+                    const info = running.get(lane.name);
+                    if (info) {
+                        info.lastOutput = data.toString().trim().split('\n').pop() || '';
+                        info.bytesReceived += data.length;
+                        // Update auto-recovery manager
+                        autoRecoveryManager.recordActivity(lane.name, data.length, info.lastOutput);
+                    }
+                });
+            }
+            const now = Date.now();
+            running.set(lane.name, {
+                ...spawnResult,
+                lastActivity: now,
+                lastStateUpdate: now,
+                stallPhase: 0,
+                taskStartTime: now,
+                lastOutput: '',
+                statePath: laneStatePath,
+                bytesReceived: 0,
+                lastBytesCheck: 0,
+                continueSignalsSent: 0,
+            });
+            // Register lane with auto-recovery manager
+            autoRecoveryManager.registerLane(lane.name);
+            // Update lane tracking
+            lane.taskStartTime = now;
             events_1.events.emit('lane.started', {
                 laneName: lane.name,
                 pid: spawnResult.child.pid,
                 logPath: spawnResult.logPath,
             });
         }
-        // 3. Wait for any running lane to finish
+        // 3. Wait for any running lane to finish OR check for stalls
         if (running.size > 0) {
+            // Polling timeout for stall detection
+            let pollTimeout;
+            const pollPromise = new Promise(resolve => {
+                pollTimeout = setTimeout(() => resolve({ name: '__poll__', code: 0 }), 10000);
+            });
             const promises = Array.from(running.entries()).map(async ([name, { child }]) => {
                 const code = await waitChild(child);
                 return { name, code };
             });
-            const finished = await Promise.race(promises);
+            const result = await Promise.race([...promises, pollPromise]);
+            if (pollTimeout)
+                clearTimeout(pollTimeout);
+            if (result.name === '__poll__') {
+                // Periodic stall check with multi-layer detection and escalating recovery
+                for (const [laneName, info] of running.entries()) {
+                    const now = Date.now();
+                    const idleTime = now - info.lastActivity;
+                    const lane = lanes.find(l => l.name === laneName);
+                    // Check state file for progress updates
+                    let progressTime = 0;
+                    try {
+                        const stateStat = fs.statSync(info.statePath);
+                        const stateUpdateTime = stateStat.mtimeMs;
+                        if (stateUpdateTime > info.lastStateUpdate) {
+                            info.lastStateUpdate = stateUpdateTime;
+                        }
+                        progressTime = now - info.lastStateUpdate;
+                    }
+                    catch {
+                        // State file might not exist yet
+                    }
+                    // Calculate bytes received since last check
+                    const bytesDelta = info.bytesReceived - info.lastBytesCheck;
+                    info.lastBytesCheck = info.bytesReceived;
+                    // Use multi-layer stall analysis with enhanced context
+                    const analysis = (0, failure_policy_1.analyzeStall)({
+                        stallPhase: info.stallPhase,
+                        idleTimeMs: idleTime,
+                        progressTimeMs: progressTime,
+                        lastOutput: info.lastOutput,
+                        restartCount: lane.restartCount || 0,
+                        taskStartTimeMs: info.taskStartTime,
+                        bytesReceived: bytesDelta, // Bytes since last check
+                        continueSignalsSent: info.continueSignalsSent,
+                    }, stallConfig);
+                    // Only act if action is not NONE
+                    if (analysis.action !== failure_policy_1.RecoveryAction.NONE) {
+                        (0, failure_policy_1.logFailure)(laneName, analysis);
+                        info.logManager?.log('error', analysis.message);
+                        if (analysis.action === failure_policy_1.RecoveryAction.CONTINUE_SIGNAL) {
+                            const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
+                            try {
+                                fs.writeFileSync(interventionPath, 'continue');
+                                info.stallPhase = 1;
+                                info.lastActivity = now;
+                                info.continueSignalsSent++;
+                                logger.info(`[${laneName}] Sent continue signal (#${info.continueSignalsSent})`);
+                                events_1.events.emit('recovery.continue_signal', {
+                                    laneName,
+                                    idleSeconds: Math.round(idleTime / 1000),
+                                    signalCount: info.continueSignalsSent,
+                                });
+                            }
+                            catch (e) {
+                                logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
+                            }
+                        }
+                        else if (analysis.action === failure_policy_1.RecoveryAction.STRONGER_PROMPT) {
+                            const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
+                            const strongerPrompt = `[SYSTEM INTERVENTION] You seem to be stuck. Please continue with your current task immediately. If you're waiting for something, explain what you need and proceed with what you can do now. If you've completed the task, summarize your work and finish.`;
+                            try {
+                                fs.writeFileSync(interventionPath, strongerPrompt);
+                                info.stallPhase = 2;
+                                info.lastActivity = now;
+                                logger.warn(`[${laneName}] Sent stronger prompt after continue signal failed`);
+                                events_1.events.emit('recovery.stronger_prompt', { laneName });
+                            }
+                            catch (e) {
+                                logger.error(`Failed to write intervention file for ${laneName}: ${e}`);
+                            }
+                        }
+                        else if (analysis.action === failure_policy_1.RecoveryAction.KILL_AND_RESTART ||
+                            analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE ||
+                            analysis.action === failure_policy_1.RecoveryAction.RESTART_LANE_FROM_CHECKPOINT) {
+                            lane.restartCount = (lane.restartCount || 0) + 1;
+                            info.stallPhase = 3;
+                            // Try to get checkpoint info
+                            const checkpoint = (0, checkpoint_1.getLatestCheckpoint)(laneRunDirs[laneName]);
+                            if (checkpoint) {
+                                logger.info(`[${laneName}] Checkpoint available: ${checkpoint.id} (task ${checkpoint.taskIndex})`);
+                            }
+                            // Kill the process
+                            try {
+                                info.child.kill('SIGKILL');
+                            }
+                            catch {
+                                // Process might already be dead
+                            }
+                            logger.warn(`[${laneName}] Killing and restarting lane (restart #${lane.restartCount})`);
+                            events_1.events.emit('recovery.restart', {
+                                laneName,
+                                restartCount: lane.restartCount,
+                                maxRestarts: stallConfig.maxRestarts,
+                            });
+                        }
+                        else if (analysis.action === failure_policy_1.RecoveryAction.RUN_DOCTOR) {
+                            info.stallPhase = 4;
+                            // Run diagnostics
+                            logger.error(`[${laneName}] Running diagnostics due to persistent failures...`);
+                            // Import health check dynamically to avoid circular dependency
+                            const { checkAgentHealth, checkAuthHealth } = await Promise.resolve().then(() => __importStar(require('../utils/health')));
+                            const [agentHealth, authHealth] = await Promise.all([
+                                checkAgentHealth(),
+                                checkAuthHealth(),
+                            ]);
+                            const issues = [];
+                            if (!agentHealth.ok)
+                                issues.push(`Agent: ${agentHealth.message}`);
+                            if (!authHealth.ok)
+                                issues.push(`Auth: ${authHealth.message}`);
+                            if (issues.length > 0) {
+                                logger.error(`[${laneName}] Diagnostic issues found:\n  ${issues.join('\n  ')}`);
+                            }
+                            else {
+                                logger.warn(`[${laneName}] No obvious issues found. The problem may be with the AI model or network.`);
+                            }
+                            // Save diagnostic to file
+                            const diagnosticPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'diagnostic.json');
+                            fs.writeFileSync(diagnosticPath, JSON.stringify({
+                                timestamp: Date.now(),
+                                agentHealthy: agentHealth.ok,
+                                authHealthy: authHealth.ok,
+                                issues,
+                                analysis,
+                            }, null, 2));
+                            // Kill the process
+                            try {
+                                info.child.kill('SIGKILL');
+                            }
+                            catch {
+                                // Process might already be dead
+                            }
+                            logger.error(`[${laneName}] Aborting lane after diagnostic. Check ${diagnosticPath} for details.`);
+                            // Save POF for failed recovery
+                            const recoveryState = autoRecoveryManager.getState(laneName);
+                            if (recoveryState) {
+                                try {
+                                    const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
+                                    const laneState = (0, state_1.loadState)(laneStatePath);
+                                    const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
+                                    const diagnosticInfo = {
+                                        timestamp: Date.now(),
+                                        agentHealthy: agentHealth.ok,
+                                        authHealthy: authHealth.ok,
+                                        systemHealthy: true,
+                                        suggestedAction: issues.length > 0 ? 'Fix the issues above and retry' : 'Try with a different model',
+                                        details: issues.join('\n') || 'No obvious issues found',
+                                    };
+                                    const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, diagnosticInfo);
+                                    (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
+                                }
+                                catch (pofError) {
+                                    logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
+                                }
+                            }
+                            events_1.events.emit('recovery.diagnosed', {
+                                laneName,
+                                diagnostic: { agentHealthy: agentHealth.ok, authHealthy: authHealth.ok, issues },
+                            });
+                        }
+                        else if (analysis.action === failure_policy_1.RecoveryAction.ABORT_LANE) {
+                            info.stallPhase = 5;
+                            try {
+                                info.child.kill('SIGKILL');
+                            }
+                            catch {
+                                // Process might already be dead
+                            }
+                            logger.error(`[${laneName}] Aborting lane due to repeated stalls`);
+                            // Save POF for failed recovery
+                            const recoveryState = autoRecoveryManager.getState(laneName);
+                            if (recoveryState) {
+                                try {
+                                    const laneStatePath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'state.json');
+                                    const laneState = (0, state_1.loadState)(laneStatePath);
+                                    const pofDir = (0, path_1.safeJoin)(runRoot, '..', '..', 'pof');
+                                    const pofEntry = (0, auto_recovery_1.createPOFFromRecoveryState)(runId, runRoot, laneName, recoveryState, laneState, recoveryState.diagnosticInfo);
+                                    (0, auto_recovery_1.savePOF)(runId, pofDir, pofEntry);
+                                }
+                                catch (pofError) {
+                                    logger.warn(`[${laneName}] Failed to save POF: ${pofError.message}`);
+                                }
+                            }
+                        }
+                        else if (analysis.action === failure_policy_1.RecoveryAction.SEND_GIT_GUIDANCE) {
+                            // Send guidance message to agent for git issues
+                            const interventionPath = (0, path_1.safeJoin)(laneRunDirs[laneName], 'intervention.txt');
+                            // Determine which guidance to send based on the failure type
+                            let guidance;
+                            if (analysis.type === failure_policy_1.FailureType.GIT_PUSH_REJECTED) {
+                                guidance = (0, auto_recovery_1.getGitPushFailureGuidance)();
+                            }
+                            else if (analysis.type === failure_policy_1.FailureType.MERGE_CONFLICT) {
+                                guidance = (0, auto_recovery_1.getMergeConflictGuidance)();
+                            }
+                            else {
+                                guidance = (0, auto_recovery_1.getGitErrorGuidance)(analysis.message);
+                            }
+                            try {
+                                fs.writeFileSync(interventionPath, guidance);
+                                info.lastActivity = now;
+                                logger.info(`[${laneName}] Sent git issue guidance to agent`);
+                            }
+                            catch (e) {
+                                logger.error(`[${laneName}] Failed to send guidance: ${e.message}`);
+                            }
+                        }
+                    }
+                }
+                continue;
+            }
+            const finished = result;
+            const info = running.get(finished.name);
             running.delete(finished.name);
             exitCodes[finished.name] = finished.code;
+            // Unregister from auto-recovery manager
+            autoRecoveryManager.unregisterLane(finished.name);
             if (finished.code === 0) {
                 completedLanes.add(finished.name);
                 events_1.events.emit('lane.completed', {
@@ -533,7 +851,7 @@ async function orchestrate(tasksDir, options = {}) {
             }
             else if (finished.code === 2) {
                 // Blocked by dependency
-                const statePath = path.join(laneRunDirs[finished.name], 'state.json');
+                const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
                 const state = (0, state_1.loadState)(statePath);
                 if (state && state.dependencyRequest) {
                     blockedLanes.set(finished.name, state.dependencyRequest);
@@ -553,11 +871,27 @@ async function orchestrate(tasksDir, options = {}) {
                 }
             }
             else {
+                // Check if it was a restart request
+                if (info.stallPhase === 2) {
+                    logger.info(`🔄 Lane ${finished.name} is being restarted due to stall...`);
+                    // Update startIndex from current state to resume from the same task
+                    const statePath = (0, path_1.safeJoin)(laneRunDirs[finished.name], 'state.json');
+                    const state = (0, state_1.loadState)(statePath);
+                    if (state) {
+                        const lane = lanes.find(l => l.name === finished.name);
+                        if (lane) {
+                            lane.startIndex = state.currentTaskIndex;
+                        }
+                    }
+                    // Note: we don't add to failedLanes or completedLanes, 
+                    // so it will be eligible to start again in the next iteration.
+                    continue;
+                }
                 failedLanes.add(finished.name);
                 events_1.events.emit('lane.failed', {
                     laneName: finished.name,
                     exitCode: finished.code,
-                    error: 'Process exited with non-zero code',
+                    error: info.stallPhase === 3 ? 'Stopped due to repeated stall' : 'Process exited with non-zero code',
                 });
             }
             printLaneStatus(lanes, laneRunDirs);