@litmers/cursorflow-orchestrator 0.1.20 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +9 -0
- package/commands/cursorflow-clean.md +19 -0
- package/commands/cursorflow-runs.md +59 -0
- package/commands/cursorflow-stop.md +55 -0
- package/dist/cli/clean.js +171 -0
- package/dist/cli/clean.js.map +1 -1
- package/dist/cli/index.js +7 -0
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.js +1 -1
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/logs.js +83 -42
- package/dist/cli/logs.js.map +1 -1
- package/dist/cli/monitor.d.ts +7 -0
- package/dist/cli/monitor.js +1007 -189
- package/dist/cli/monitor.js.map +1 -1
- package/dist/cli/prepare.js +4 -3
- package/dist/cli/prepare.js.map +1 -1
- package/dist/cli/resume.js +188 -236
- package/dist/cli/resume.js.map +1 -1
- package/dist/cli/run.js +8 -3
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/runs.d.ts +5 -0
- package/dist/cli/runs.js +214 -0
- package/dist/cli/runs.js.map +1 -0
- package/dist/cli/setup-commands.js +0 -0
- package/dist/cli/signal.js +1 -1
- package/dist/cli/signal.js.map +1 -1
- package/dist/cli/stop.d.ts +5 -0
- package/dist/cli/stop.js +215 -0
- package/dist/cli/stop.js.map +1 -0
- package/dist/cli/tasks.d.ts +10 -0
- package/dist/cli/tasks.js +165 -0
- package/dist/cli/tasks.js.map +1 -0
- package/dist/core/auto-recovery.d.ts +212 -0
- package/dist/core/auto-recovery.js +737 -0
- package/dist/core/auto-recovery.js.map +1 -0
- package/dist/core/failure-policy.d.ts +156 -0
- package/dist/core/failure-policy.js +488 -0
- package/dist/core/failure-policy.js.map +1 -0
- package/dist/core/orchestrator.d.ts +15 -2
- package/dist/core/orchestrator.js +392 -15
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/reviewer.d.ts +2 -0
- package/dist/core/reviewer.js +2 -0
- package/dist/core/reviewer.js.map +1 -1
- package/dist/core/runner.d.ts +33 -10
- package/dist/core/runner.js +321 -146
- package/dist/core/runner.js.map +1 -1
- package/dist/services/logging/buffer.d.ts +67 -0
- package/dist/services/logging/buffer.js +309 -0
- package/dist/services/logging/buffer.js.map +1 -0
- package/dist/services/logging/console.d.ts +89 -0
- package/dist/services/logging/console.js +169 -0
- package/dist/services/logging/console.js.map +1 -0
- package/dist/services/logging/file-writer.d.ts +71 -0
- package/dist/services/logging/file-writer.js +516 -0
- package/dist/services/logging/file-writer.js.map +1 -0
- package/dist/services/logging/formatter.d.ts +39 -0
- package/dist/services/logging/formatter.js +227 -0
- package/dist/services/logging/formatter.js.map +1 -0
- package/dist/services/logging/index.d.ts +11 -0
- package/dist/services/logging/index.js +30 -0
- package/dist/services/logging/index.js.map +1 -0
- package/dist/services/logging/parser.d.ts +31 -0
- package/dist/services/logging/parser.js +222 -0
- package/dist/services/logging/parser.js.map +1 -0
- package/dist/services/process/index.d.ts +59 -0
- package/dist/services/process/index.js +257 -0
- package/dist/services/process/index.js.map +1 -0
- package/dist/types/agent.d.ts +20 -0
- package/dist/types/agent.js +6 -0
- package/dist/types/agent.js.map +1 -0
- package/dist/types/config.d.ts +65 -0
- package/dist/types/config.js +6 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/events.d.ts +125 -0
- package/dist/types/events.js +6 -0
- package/dist/types/events.js.map +1 -0
- package/dist/types/index.d.ts +12 -0
- package/dist/types/index.js +37 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/lane.d.ts +43 -0
- package/dist/types/lane.js +6 -0
- package/dist/types/lane.js.map +1 -0
- package/dist/types/logging.d.ts +71 -0
- package/dist/types/logging.js +16 -0
- package/dist/types/logging.js.map +1 -0
- package/dist/types/review.d.ts +17 -0
- package/dist/types/review.js +6 -0
- package/dist/types/review.js.map +1 -0
- package/dist/types/run.d.ts +32 -0
- package/dist/types/run.js +6 -0
- package/dist/types/run.js.map +1 -0
- package/dist/types/task.d.ts +71 -0
- package/dist/types/task.js +6 -0
- package/dist/types/task.js.map +1 -0
- package/dist/ui/components.d.ts +134 -0
- package/dist/ui/components.js +389 -0
- package/dist/ui/components.js.map +1 -0
- package/dist/ui/log-viewer.d.ts +49 -0
- package/dist/ui/log-viewer.js +449 -0
- package/dist/ui/log-viewer.js.map +1 -0
- package/dist/utils/checkpoint.d.ts +87 -0
- package/dist/utils/checkpoint.js +317 -0
- package/dist/utils/checkpoint.js.map +1 -0
- package/dist/utils/config.d.ts +4 -0
- package/dist/utils/config.js +11 -2
- package/dist/utils/config.js.map +1 -1
- package/dist/utils/cursor-agent.js.map +1 -1
- package/dist/utils/dependency.d.ts +74 -0
- package/dist/utils/dependency.js +420 -0
- package/dist/utils/dependency.js.map +1 -0
- package/dist/utils/doctor.js +10 -5
- package/dist/utils/doctor.js.map +1 -1
- package/dist/utils/enhanced-logger.d.ts +10 -33
- package/dist/utils/enhanced-logger.js +94 -9
- package/dist/utils/enhanced-logger.js.map +1 -1
- package/dist/utils/git.d.ts +121 -0
- package/dist/utils/git.js +322 -2
- package/dist/utils/git.js.map +1 -1
- package/dist/utils/health.d.ts +91 -0
- package/dist/utils/health.js +556 -0
- package/dist/utils/health.js.map +1 -0
- package/dist/utils/lock.d.ts +95 -0
- package/dist/utils/lock.js +332 -0
- package/dist/utils/lock.js.map +1 -0
- package/dist/utils/log-buffer.d.ts +17 -0
- package/dist/utils/log-buffer.js +14 -0
- package/dist/utils/log-buffer.js.map +1 -0
- package/dist/utils/log-constants.d.ts +23 -0
- package/dist/utils/log-constants.js +28 -0
- package/dist/utils/log-constants.js.map +1 -0
- package/dist/utils/log-formatter.d.ts +9 -0
- package/dist/utils/log-formatter.js +113 -70
- package/dist/utils/log-formatter.js.map +1 -1
- package/dist/utils/log-service.d.ts +19 -0
- package/dist/utils/log-service.js +47 -0
- package/dist/utils/log-service.js.map +1 -0
- package/dist/utils/logger.d.ts +46 -27
- package/dist/utils/logger.js +82 -60
- package/dist/utils/logger.js.map +1 -1
- package/dist/utils/process-manager.d.ts +21 -0
- package/dist/utils/process-manager.js +138 -0
- package/dist/utils/process-manager.js.map +1 -0
- package/dist/utils/retry.d.ts +121 -0
- package/dist/utils/retry.js +374 -0
- package/dist/utils/retry.js.map +1 -0
- package/dist/utils/run-service.d.ts +88 -0
- package/dist/utils/run-service.js +412 -0
- package/dist/utils/run-service.js.map +1 -0
- package/dist/utils/state.d.ts +58 -2
- package/dist/utils/state.js +306 -3
- package/dist/utils/state.js.map +1 -1
- package/dist/utils/task-service.d.ts +82 -0
- package/dist/utils/task-service.js +348 -0
- package/dist/utils/task-service.js.map +1 -0
- package/dist/utils/types.d.ts +2 -272
- package/dist/utils/types.js +16 -0
- package/dist/utils/types.js.map +1 -1
- package/package.json +38 -23
- package/scripts/ai-security-check.js +0 -1
- package/scripts/local-security-gate.sh +0 -0
- package/scripts/monitor-lanes.sh +94 -0
- package/scripts/patches/test-cursor-agent.js +0 -1
- package/scripts/release.sh +0 -0
- package/scripts/setup-security.sh +0 -0
- package/scripts/stream-logs.sh +72 -0
- package/scripts/verify-and-fix.sh +0 -0
- package/src/cli/clean.ts +180 -0
- package/src/cli/index.ts +7 -0
- package/src/cli/init.ts +1 -1
- package/src/cli/logs.ts +79 -42
- package/src/cli/monitor.ts +1815 -899
- package/src/cli/prepare.ts +4 -3
- package/src/cli/resume.ts +220 -277
- package/src/cli/run.ts +9 -3
- package/src/cli/runs.ts +212 -0
- package/src/cli/setup-commands.ts +0 -0
- package/src/cli/signal.ts +1 -1
- package/src/cli/stop.ts +209 -0
- package/src/cli/tasks.ts +154 -0
- package/src/core/auto-recovery.ts +909 -0
- package/src/core/failure-policy.ts +592 -0
- package/src/core/orchestrator.ts +1131 -675
- package/src/core/reviewer.ts +4 -0
- package/src/core/runner.ts +388 -162
- package/src/services/logging/buffer.ts +326 -0
- package/src/services/logging/console.ts +193 -0
- package/src/services/logging/file-writer.ts +526 -0
- package/src/services/logging/formatter.ts +268 -0
- package/src/services/logging/index.ts +16 -0
- package/src/services/logging/parser.ts +232 -0
- package/src/services/process/index.ts +261 -0
- package/src/types/agent.ts +24 -0
- package/src/types/config.ts +79 -0
- package/src/types/events.ts +156 -0
- package/src/types/index.ts +29 -0
- package/src/types/lane.ts +56 -0
- package/src/types/logging.ts +96 -0
- package/src/types/review.ts +20 -0
- package/src/types/run.ts +37 -0
- package/src/types/task.ts +79 -0
- package/src/ui/components.ts +430 -0
- package/src/ui/log-viewer.ts +485 -0
- package/src/utils/checkpoint.ts +374 -0
- package/src/utils/config.ts +11 -2
- package/src/utils/cursor-agent.ts +1 -1
- package/src/utils/dependency.ts +482 -0
- package/src/utils/doctor.ts +11 -5
- package/src/utils/enhanced-logger.ts +108 -49
- package/src/utils/git.ts +374 -2
- package/src/utils/health.ts +596 -0
- package/src/utils/lock.ts +346 -0
- package/src/utils/log-buffer.ts +28 -0
- package/src/utils/log-constants.ts +26 -0
- package/src/utils/log-formatter.ts +120 -37
- package/src/utils/log-service.ts +49 -0
- package/src/utils/logger.ts +100 -51
- package/src/utils/process-manager.ts +100 -0
- package/src/utils/retry.ts +413 -0
- package/src/utils/run-service.ts +433 -0
- package/src/utils/state.ts +369 -3
- package/src/utils/task-service.ts +370 -0
- package/src/utils/types.ts +2 -315
package/src/cli/resume.ts
CHANGED
|
@@ -6,18 +6,17 @@ import * as path from 'path';
|
|
|
6
6
|
import * as fs from 'fs';
|
|
7
7
|
import { spawn, ChildProcess } from 'child_process';
|
|
8
8
|
import * as logger from '../utils/logger';
|
|
9
|
-
import { loadConfig, getLogsDir } from '../utils/config';
|
|
10
|
-
import { loadState } from '../utils/state';
|
|
11
|
-
import { LaneState } from '../
|
|
9
|
+
import { loadConfig, getLogsDir, getPofDir } from '../utils/config';
|
|
10
|
+
import { loadState, saveState } from '../utils/state';
|
|
11
|
+
import { LaneState } from '../types';
|
|
12
12
|
import { runDoctor } from '../utils/doctor';
|
|
13
13
|
import { safeJoin } from '../utils/path';
|
|
14
14
|
import {
|
|
15
15
|
EnhancedLogManager,
|
|
16
16
|
createLogManager,
|
|
17
|
-
DEFAULT_LOG_CONFIG,
|
|
18
|
-
stripAnsi,
|
|
19
17
|
ParsedMessage
|
|
20
18
|
} from '../utils/enhanced-logger';
|
|
19
|
+
import { formatMessageForConsole } from '../utils/log-formatter';
|
|
21
20
|
|
|
22
21
|
interface ResumeOptions {
|
|
23
22
|
lane: string | null;
|
|
@@ -40,7 +39,7 @@ Usage: cursorflow resume [lane] [options]
|
|
|
40
39
|
Resume interrupted or failed lanes.
|
|
41
40
|
|
|
42
41
|
Options:
|
|
43
|
-
<lane> Lane name
|
|
42
|
+
<lane> Lane name or tasks directory to resume
|
|
44
43
|
--all Resume ALL incomplete/failed lanes
|
|
45
44
|
--status Show status of all lanes in the run (no resume)
|
|
46
45
|
--run-dir <path> Use a specific run directory (default: latest)
|
|
@@ -56,8 +55,8 @@ Examples:
|
|
|
56
55
|
cursorflow resume --status # Check status of all lanes
|
|
57
56
|
cursorflow resume --all # Resume all incomplete lanes
|
|
58
57
|
cursorflow resume lane-1 # Resume single lane
|
|
58
|
+
cursorflow resume _cursorflow/tasks/feat1 # Resume all lanes in directory
|
|
59
59
|
cursorflow resume --all --restart # Restart all incomplete lanes from task 0
|
|
60
|
-
cursorflow resume --all --max-concurrent 2 # Resume with max 2 parallel lanes
|
|
61
60
|
`);
|
|
62
61
|
}
|
|
63
62
|
|
|
@@ -111,86 +110,167 @@ const STATUS_COLORS: Record<string, string> = {
|
|
|
111
110
|
};
|
|
112
111
|
const RESET = '\x1b[0m';
|
|
113
112
|
|
|
113
|
+
interface LaneInfo {
|
|
114
|
+
name: string;
|
|
115
|
+
dir: string;
|
|
116
|
+
state: LaneState | null;
|
|
117
|
+
needsResume: boolean;
|
|
118
|
+
dependsOn: string[];
|
|
119
|
+
isCompleted: boolean;
|
|
120
|
+
}
|
|
121
|
+
|
|
114
122
|
/**
|
|
115
|
-
*
|
|
123
|
+
* Check if a process is alive by its PID
|
|
116
124
|
*/
|
|
117
|
-
function
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
if (toolMatch) {
|
|
136
|
-
const [, name, args] = toolMatch;
|
|
137
|
-
try {
|
|
138
|
-
const parsedArgs = JSON.parse(args!);
|
|
139
|
-
let argStr = '';
|
|
140
|
-
if (name === 'read_file' && parsedArgs.target_file) argStr = parsedArgs.target_file;
|
|
141
|
-
else if (name === 'run_terminal_cmd' && parsedArgs.command) argStr = parsedArgs.command;
|
|
142
|
-
else if (name === 'write' && parsedArgs.file_path) argStr = parsedArgs.file_path;
|
|
143
|
-
else if (name === 'search_replace' && parsedArgs.file_path) argStr = parsedArgs.file_path;
|
|
144
|
-
else {
|
|
145
|
-
const keys = Object.keys(parsedArgs);
|
|
146
|
-
if (keys.length > 0) argStr = String(parsedArgs[keys[0]]).substring(0, 50);
|
|
147
|
-
}
|
|
148
|
-
content = `${logger.COLORS.bold}${name}${logger.COLORS.reset}(${argStr})`;
|
|
149
|
-
} catch {
|
|
150
|
-
content = `${logger.COLORS.bold}${name}${logger.COLORS.reset}: ${args}`;
|
|
151
|
-
}
|
|
152
|
-
}
|
|
153
|
-
break;
|
|
154
|
-
case 'tool_result':
|
|
155
|
-
prefix = `${logger.COLORS.gray}📄 RESL${logger.COLORS.reset}`;
|
|
156
|
-
const resMatch = content.match(/\[Tool Result: ([^\]]+)\]/);
|
|
157
|
-
content = resMatch ? `${resMatch[1]} OK` : 'result';
|
|
158
|
-
break;
|
|
159
|
-
case 'result':
|
|
160
|
-
prefix = `${logger.COLORS.green}✅ DONE${logger.COLORS.reset}`;
|
|
161
|
-
break;
|
|
162
|
-
case 'system':
|
|
163
|
-
prefix = `${logger.COLORS.gray}⚙️ SYS${logger.COLORS.reset}`;
|
|
164
|
-
break;
|
|
165
|
-
case 'thinking':
|
|
166
|
-
prefix = `${logger.COLORS.gray}🤔 THNK${logger.COLORS.reset}`;
|
|
167
|
-
break;
|
|
125
|
+
function isProcessAlive(pid: number): boolean {
|
|
126
|
+
try {
|
|
127
|
+
// On Unix-like systems, sending signal 0 checks if process exists
|
|
128
|
+
process.kill(pid, 0);
|
|
129
|
+
return true;
|
|
130
|
+
} catch {
|
|
131
|
+
return false;
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* Check for zombie "running" lanes and fix them
|
|
137
|
+
* A zombie lane is one that has status "running" but its process is dead
|
|
138
|
+
*/
|
|
139
|
+
function checkAndFixZombieLanes(runDir: string): { fixed: string[]; pofCreated: boolean } {
|
|
140
|
+
const lanesDir = safeJoin(runDir, 'lanes');
|
|
141
|
+
if (!fs.existsSync(lanesDir)) {
|
|
142
|
+
return { fixed: [], pofCreated: false };
|
|
168
143
|
}
|
|
169
144
|
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
145
|
+
const fixed: string[] = [];
|
|
146
|
+
const zombieDetails: Array<{
|
|
147
|
+
name: string;
|
|
148
|
+
pid: number;
|
|
149
|
+
taskIndex: number;
|
|
150
|
+
totalTasks: number;
|
|
151
|
+
}> = [];
|
|
152
|
+
|
|
153
|
+
const laneDirs = fs.readdirSync(lanesDir)
|
|
154
|
+
.filter(f => fs.statSync(safeJoin(lanesDir, f)).isDirectory());
|
|
155
|
+
|
|
156
|
+
for (const laneName of laneDirs) {
|
|
157
|
+
const dir = safeJoin(lanesDir, laneName);
|
|
158
|
+
const statePath = safeJoin(dir, 'state.json');
|
|
159
|
+
|
|
160
|
+
if (!fs.existsSync(statePath)) continue;
|
|
161
|
+
|
|
162
|
+
const state = loadState<LaneState>(statePath);
|
|
163
|
+
if (!state) continue;
|
|
173
164
|
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
165
|
+
// Check for zombie: status is "running" but process is dead
|
|
166
|
+
if (state.status === 'running' && state.pid) {
|
|
167
|
+
const alive = isProcessAlive(state.pid);
|
|
168
|
+
|
|
169
|
+
if (!alive) {
|
|
170
|
+
logger.warn(`🧟 Zombie lane detected: ${laneName} (PID ${state.pid} is dead)`);
|
|
171
|
+
|
|
172
|
+
// Update state to failed
|
|
173
|
+
const updatedState: LaneState = {
|
|
174
|
+
...state,
|
|
175
|
+
status: 'failed',
|
|
176
|
+
error: `Process terminated unexpectedly (PID ${state.pid} was running but is now dead)`,
|
|
177
|
+
endTime: Date.now(),
|
|
178
|
+
};
|
|
179
|
+
|
|
180
|
+
saveState(statePath, updatedState);
|
|
181
|
+
fixed.push(laneName);
|
|
182
|
+
|
|
183
|
+
zombieDetails.push({
|
|
184
|
+
name: laneName,
|
|
185
|
+
pid: state.pid,
|
|
186
|
+
taskIndex: state.currentTaskIndex,
|
|
187
|
+
totalTasks: state.totalTasks,
|
|
188
|
+
});
|
|
189
|
+
|
|
190
|
+
logger.info(` → Status changed to 'failed', ready for resume`);
|
|
179
191
|
}
|
|
180
|
-
process.stdout.write(`${tsPrefix} ${' '.repeat(stripAnsi(prefix).length)} └${'─'.repeat(60)}\n`);
|
|
181
|
-
} else {
|
|
182
|
-
process.stdout.write(`${tsPrefix} ${prefix} ${content}\n`);
|
|
183
192
|
}
|
|
184
193
|
}
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
+
|
|
195
|
+
// Create POF file if any zombies were found
|
|
196
|
+
let pofCreated = false;
|
|
197
|
+
if (zombieDetails.length > 0) {
|
|
198
|
+
const config = loadConfig();
|
|
199
|
+
const pofDir = getPofDir(config);
|
|
200
|
+
if (!fs.existsSync(pofDir)) {
|
|
201
|
+
fs.mkdirSync(pofDir, { recursive: true });
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
const runId = path.basename(runDir);
|
|
205
|
+
const pofPath = safeJoin(pofDir, `pof-${runId}.json`);
|
|
206
|
+
|
|
207
|
+
let existingPof = null;
|
|
208
|
+
try {
|
|
209
|
+
existingPof = JSON.parse(fs.readFileSync(pofPath, 'utf-8'));
|
|
210
|
+
} catch {
|
|
211
|
+
// Ignore errors (file might not exist)
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
const pof = {
|
|
215
|
+
title: 'Run Failure Post-mortem',
|
|
216
|
+
runId: path.basename(runDir),
|
|
217
|
+
failureTime: new Date().toISOString(),
|
|
218
|
+
detectedAt: new Date().toISOString(),
|
|
219
|
+
summary: `${zombieDetails.length} lane(s) found with dead processes (zombie state)`,
|
|
220
|
+
|
|
221
|
+
rootCause: {
|
|
222
|
+
type: 'ZOMBIE_PROCESS',
|
|
223
|
+
description: 'Lane processes were marked as running but the processes are no longer alive',
|
|
224
|
+
symptoms: [
|
|
225
|
+
'Process PIDs no longer exist in the system',
|
|
226
|
+
'Lanes were stuck in "running" state',
|
|
227
|
+
'No completion or error was recorded before process death',
|
|
228
|
+
],
|
|
229
|
+
},
|
|
230
|
+
|
|
231
|
+
affectedLanes: zombieDetails.map(z => ({
|
|
232
|
+
name: z.name,
|
|
233
|
+
status: 'failed (was: running)',
|
|
234
|
+
task: `[${z.taskIndex + 1}/${z.totalTasks}]`,
|
|
235
|
+
taskIndex: z.taskIndex,
|
|
236
|
+
pid: z.pid,
|
|
237
|
+
reason: 'Process terminated unexpectedly',
|
|
238
|
+
})),
|
|
239
|
+
|
|
240
|
+
possibleCauses: [
|
|
241
|
+
'System killed process due to memory pressure (OOM)',
|
|
242
|
+
'User killed process manually (Ctrl+C, kill command)',
|
|
243
|
+
'Agent timeout exceeded and process was terminated',
|
|
244
|
+
'System restart or crash',
|
|
245
|
+
'Agent hung and watchdog terminated it',
|
|
246
|
+
],
|
|
247
|
+
|
|
248
|
+
recovery: {
|
|
249
|
+
command: `cursorflow resume --all --run-dir ${runDir}`,
|
|
250
|
+
description: 'Resume all failed lanes from their last checkpoint',
|
|
251
|
+
alternativeCommand: `cursorflow resume --all --restart --run-dir ${runDir}`,
|
|
252
|
+
alternativeDescription: 'Restart all failed lanes from the beginning',
|
|
253
|
+
},
|
|
254
|
+
|
|
255
|
+
// Merge with existing POF if present
|
|
256
|
+
previousFailures: existingPof ? [existingPof] : undefined,
|
|
257
|
+
};
|
|
258
|
+
|
|
259
|
+
// Use atomic write: write to temp file then rename
|
|
260
|
+
const tempPath = `${pofPath}.${Math.random().toString(36).substring(2, 7)}.tmp`;
|
|
261
|
+
try {
|
|
262
|
+
fs.writeFileSync(tempPath, JSON.stringify(pof, null, 2), 'utf8');
|
|
263
|
+
fs.renameSync(tempPath, pofPath);
|
|
264
|
+
pofCreated = true;
|
|
265
|
+
logger.info(`📋 POF file created: ${pofPath}`);
|
|
266
|
+
} catch (err) {
|
|
267
|
+
// If temp file was created, try to clean it up
|
|
268
|
+
try { if (fs.existsSync(tempPath)) fs.unlinkSync(tempPath); } catch { /* ignore */ }
|
|
269
|
+
throw err;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
return { fixed, pofCreated };
|
|
194
274
|
}
|
|
195
275
|
|
|
196
276
|
/**
|
|
@@ -384,56 +464,28 @@ function spawnLaneResume(
|
|
|
384
464
|
runnerArgs.push('--executor', options.executor);
|
|
385
465
|
}
|
|
386
466
|
|
|
387
|
-
const logManager = createLogManager(laneDir, laneName, options.enhancedLogConfig || {}, (msg) =>
|
|
467
|
+
const logManager = createLogManager(laneDir, laneName, options.enhancedLogConfig || {}, (msg) => {
|
|
468
|
+
const formatted = formatMessageForConsole(msg, {
|
|
469
|
+
laneLabel: `[${laneName}]`,
|
|
470
|
+
includeTimestamp: true
|
|
471
|
+
});
|
|
472
|
+
process.stdout.write(formatted + '\n');
|
|
473
|
+
});
|
|
388
474
|
|
|
389
475
|
const child = spawn('node', runnerArgs, {
|
|
390
476
|
stdio: ['ignore', 'pipe', 'pipe'],
|
|
391
477
|
env: process.env,
|
|
392
478
|
});
|
|
393
479
|
|
|
394
|
-
let lineBuffer = '';
|
|
395
|
-
|
|
396
480
|
if (child.stdout) {
|
|
397
481
|
child.stdout.on('data', (data: Buffer) => {
|
|
398
482
|
logManager.writeStdout(data);
|
|
399
|
-
|
|
400
|
-
const str = data.toString();
|
|
401
|
-
lineBuffer += str;
|
|
402
|
-
const lines = lineBuffer.split('\n');
|
|
403
|
-
lineBuffer = lines.pop() || '';
|
|
404
|
-
|
|
405
|
-
for (const line of lines) {
|
|
406
|
-
const trimmed = line.trim();
|
|
407
|
-
if (trimmed &&
|
|
408
|
-
!trimmed.startsWith('{') &&
|
|
409
|
-
!trimmed.startsWith('[') &&
|
|
410
|
-
!trimmed.includes('{"type"')) {
|
|
411
|
-
process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${logger.COLORS.magenta}${laneName.padEnd(10)}${logger.COLORS.reset} ${line}\n`);
|
|
412
|
-
}
|
|
413
|
-
}
|
|
414
483
|
});
|
|
415
484
|
}
|
|
416
485
|
|
|
417
486
|
if (child.stderr) {
|
|
418
487
|
child.stderr.on('data', (data: Buffer) => {
|
|
419
488
|
logManager.writeStderr(data);
|
|
420
|
-
const str = data.toString();
|
|
421
|
-
const lines = str.split('\n');
|
|
422
|
-
for (const line of lines) {
|
|
423
|
-
const trimmed = line.trim();
|
|
424
|
-
if (trimmed) {
|
|
425
|
-
const isStatus = trimmed.startsWith('Preparing worktree') ||
|
|
426
|
-
trimmed.startsWith('Switched to a new branch') ||
|
|
427
|
-
trimmed.startsWith('HEAD is now at') ||
|
|
428
|
-
trimmed.includes('actual output');
|
|
429
|
-
|
|
430
|
-
if (isStatus) {
|
|
431
|
-
process.stdout.write(`${logger.COLORS.gray}[${new Date().toLocaleTimeString('en-US', { hour12: false })}]${logger.COLORS.reset} ${logger.COLORS.magenta}${laneName.padEnd(10)}${logger.COLORS.reset} ${trimmed}\n`);
|
|
432
|
-
} else {
|
|
433
|
-
process.stderr.write(`${logger.COLORS.red}[${laneName}] ERROR: ${trimmed}${logger.COLORS.reset}\n`);
|
|
434
|
-
}
|
|
435
|
-
}
|
|
436
|
-
}
|
|
437
489
|
});
|
|
438
490
|
}
|
|
439
491
|
|
|
@@ -461,8 +513,9 @@ function waitForChild(child: ChildProcess): Promise<number> {
|
|
|
461
513
|
/**
|
|
462
514
|
* Resume multiple lanes with concurrency control and dependency awareness
|
|
463
515
|
*/
|
|
464
|
-
async function
|
|
465
|
-
|
|
516
|
+
async function resumeLanes(
|
|
517
|
+
lanesToResume: LaneInfo[],
|
|
518
|
+
allLanes: LaneInfo[],
|
|
466
519
|
options: {
|
|
467
520
|
restart: boolean;
|
|
468
521
|
maxConcurrent: number;
|
|
@@ -472,21 +525,6 @@ async function resumeAllLanes(
|
|
|
472
525
|
enhancedLogConfig?: any;
|
|
473
526
|
}
|
|
474
527
|
): Promise<{ succeeded: string[]; failed: string[]; skipped: string[] }> {
|
|
475
|
-
const allLanes = getAllLaneStatuses(runDir);
|
|
476
|
-
const lanesToResume = allLanes.filter(l => l.needsResume && l.state?.tasksFile);
|
|
477
|
-
const missingTaskInfo = allLanes.filter(l => l.needsResume && !l.state?.tasksFile);
|
|
478
|
-
|
|
479
|
-
if (missingTaskInfo.length > 0) {
|
|
480
|
-
logger.warn(`Lanes that haven't started yet and have no task info: ${missingTaskInfo.map(l => l.name).join(', ')}`);
|
|
481
|
-
logger.warn('These lanes cannot be resumed because their original task file paths were not recorded.');
|
|
482
|
-
}
|
|
483
|
-
|
|
484
|
-
if (lanesToResume.length === 0) {
|
|
485
|
-
logger.success('All lanes are already completed! Nothing to resume.');
|
|
486
|
-
return { succeeded: [], failed: [], skipped: [] };
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
// Check for lanes with unmet dependencies that can never be satisfied
|
|
490
528
|
const completedSet = new Set<string>(allLanes.filter(l => l.isCompleted).map(l => l.name));
|
|
491
529
|
const toResumeNames = new Set<string>(lanesToResume.map(l => l.name));
|
|
492
530
|
|
|
@@ -516,18 +554,9 @@ async function resumeAllLanes(
|
|
|
516
554
|
logger.info(`Max concurrent: ${options.maxConcurrent}`);
|
|
517
555
|
logger.info(`Mode: ${options.restart ? 'Restart from beginning' : 'Continue from last task'}`);
|
|
518
556
|
|
|
519
|
-
// Show dependency order
|
|
520
|
-
const lanesWithDeps = resolvableLanes.filter(l => l.dependsOn.length > 0);
|
|
521
|
-
if (lanesWithDeps.length > 0) {
|
|
522
|
-
logger.info(`Dependency-aware: ${lanesWithDeps.length} lane(s) have dependencies`);
|
|
523
|
-
}
|
|
524
|
-
console.log('');
|
|
525
|
-
|
|
526
557
|
// Run doctor check once if needed (check git status)
|
|
527
558
|
if (!options.skipDoctor) {
|
|
528
559
|
logger.info('Running pre-flight checks...');
|
|
529
|
-
|
|
530
|
-
// Use the first lane's tasksDir for doctor check
|
|
531
560
|
const firstLane = resolvableLanes[0]!;
|
|
532
561
|
const tasksDir = path.dirname(firstLane.state!.tasksFile!);
|
|
533
562
|
|
|
@@ -554,18 +583,11 @@ async function resumeAllLanes(
|
|
|
554
583
|
|
|
555
584
|
const succeeded: string[] = [];
|
|
556
585
|
const failed: string[] = [];
|
|
557
|
-
|
|
558
|
-
// Create a mutable set for tracking completed lanes (including those from this session)
|
|
559
586
|
const sessionCompleted = new Set<string>(completedSet);
|
|
560
|
-
|
|
561
|
-
// Queue management with dependency awareness
|
|
562
587
|
const pending = new Set<string>(resolvableLanes.map(l => l.name));
|
|
563
588
|
const active: Map<string, ChildProcess> = new Map();
|
|
564
589
|
const laneMap = new Map<string, LaneInfo>(resolvableLanes.map(l => [l.name, l]));
|
|
565
590
|
|
|
566
|
-
/**
|
|
567
|
-
* Find the next lane that can be started (all dependencies met)
|
|
568
|
-
*/
|
|
569
591
|
const findReadyLane = (): LaneInfo | null => {
|
|
570
592
|
for (const laneName of pending) {
|
|
571
593
|
const lane = laneMap.get(laneName)!;
|
|
@@ -576,29 +598,20 @@ async function resumeAllLanes(
|
|
|
576
598
|
return null;
|
|
577
599
|
};
|
|
578
600
|
|
|
579
|
-
/**
|
|
580
|
-
* Process lanes with dependency awareness
|
|
581
|
-
*/
|
|
582
601
|
const processNext = (): void => {
|
|
583
602
|
while (active.size < options.maxConcurrent) {
|
|
584
603
|
const lane = findReadyLane();
|
|
585
|
-
|
|
586
604
|
if (!lane) {
|
|
587
|
-
// No lane ready to start
|
|
588
605
|
if (pending.size > 0 && active.size === 0) {
|
|
589
|
-
// Deadlock: pending lanes exist but none can start and none are running
|
|
590
606
|
const pendingList = Array.from(pending).join(', ');
|
|
591
607
|
logger.error(`Deadlock detected! Lanes waiting: ${pendingList}`);
|
|
592
|
-
for (const ln of pending)
|
|
593
|
-
failed.push(ln);
|
|
594
|
-
}
|
|
608
|
+
for (const ln of pending) failed.push(ln);
|
|
595
609
|
pending.clear();
|
|
596
610
|
}
|
|
597
611
|
break;
|
|
598
612
|
}
|
|
599
613
|
|
|
600
614
|
pending.delete(lane.name);
|
|
601
|
-
|
|
602
615
|
const depsInfo = lane.dependsOn.length > 0 ? ` (after: ${lane.dependsOn.join(', ')})` : '';
|
|
603
616
|
logger.info(`Starting: ${lane.name} (task ${lane.state!.currentTaskIndex}/${lane.state!.totalTasks})${depsInfo}`);
|
|
604
617
|
|
|
@@ -611,14 +624,12 @@ async function resumeAllLanes(
|
|
|
611
624
|
|
|
612
625
|
active.set(lane.name, child);
|
|
613
626
|
|
|
614
|
-
// Handle completion
|
|
615
627
|
waitForChild(child).then(code => {
|
|
616
628
|
active.delete(lane.name);
|
|
617
|
-
|
|
618
629
|
if (code === 0) {
|
|
619
630
|
logger.success(`✓ ${lane.name} completed`);
|
|
620
631
|
succeeded.push(lane.name);
|
|
621
|
-
sessionCompleted.add(lane.name);
|
|
632
|
+
sessionCompleted.add(lane.name);
|
|
622
633
|
} else if (code === 2) {
|
|
623
634
|
logger.warn(`⚠ ${lane.name} blocked on dependency change`);
|
|
624
635
|
failed.push(lane.name);
|
|
@@ -626,8 +637,6 @@ async function resumeAllLanes(
|
|
|
626
637
|
logger.error(`✗ ${lane.name} failed (exit ${code})`);
|
|
627
638
|
failed.push(lane.name);
|
|
628
639
|
}
|
|
629
|
-
|
|
630
|
-
// Try to start more lanes now that one completed
|
|
631
640
|
processNext();
|
|
632
641
|
}).catch(err => {
|
|
633
642
|
active.delete(lane.name);
|
|
@@ -638,29 +647,20 @@ async function resumeAllLanes(
|
|
|
638
647
|
}
|
|
639
648
|
};
|
|
640
649
|
|
|
641
|
-
// Start initial batch
|
|
642
650
|
processNext();
|
|
643
651
|
|
|
644
|
-
// Wait for all to complete
|
|
645
652
|
while (active.size > 0 || pending.size > 0) {
|
|
646
653
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
647
|
-
|
|
648
|
-
// Check if we can start more (in case completion handlers haven't triggered processNext yet)
|
|
649
654
|
if (active.size < options.maxConcurrent && pending.size > 0) {
|
|
650
655
|
processNext();
|
|
651
656
|
}
|
|
652
657
|
}
|
|
653
658
|
|
|
654
|
-
// Summary
|
|
655
659
|
console.log('');
|
|
656
660
|
logger.section('📊 Resume Summary');
|
|
657
661
|
logger.info(`Succeeded: ${succeeded.length}`);
|
|
658
|
-
if (failed.length > 0) {
|
|
659
|
-
|
|
660
|
-
}
|
|
661
|
-
if (skippedLanes.length > 0) {
|
|
662
|
-
logger.warn(`Skipped: ${skippedLanes.length} (${skippedLanes.join(', ')})`);
|
|
663
|
-
}
|
|
662
|
+
if (failed.length > 0) logger.error(`Failed: ${failed.length} (${failed.join(', ')})`);
|
|
663
|
+
if (skippedLanes.length > 0) logger.warn(`Skipped: ${skippedLanes.length} (${skippedLanes.join(', ')})`);
|
|
664
664
|
|
|
665
665
|
return { succeeded, failed, skipped: skippedLanes };
|
|
666
666
|
}
|
|
@@ -676,7 +676,6 @@ async function resume(args: string[]): Promise<void> {
|
|
|
676
676
|
const config = loadConfig();
|
|
677
677
|
const logsDir = getLogsDir(config);
|
|
678
678
|
|
|
679
|
-
// Find run directory
|
|
680
679
|
let runDir = options.runDir;
|
|
681
680
|
if (!runDir) {
|
|
682
681
|
runDir = findLatestRunDir(logsDir);
|
|
@@ -686,125 +685,69 @@ async function resume(args: string[]): Promise<void> {
|
|
|
686
685
|
throw new Error(`Run directory not found: ${runDir || 'latest'}. Have you run any tasks yet?`);
|
|
687
686
|
}
|
|
688
687
|
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
if (options.all) {
|
|
697
|
-
const result = await resumeAllLanes(runDir, {
|
|
698
|
-
restart: options.restart,
|
|
699
|
-
maxConcurrent: options.maxConcurrent,
|
|
700
|
-
skipDoctor: options.skipDoctor,
|
|
701
|
-
noGit: options.noGit,
|
|
702
|
-
executor: options.executor,
|
|
703
|
-
enhancedLogConfig: config.enhancedLogging,
|
|
704
|
-
});
|
|
688
|
+
const allLanes = getAllLaneStatuses(runDir);
|
|
689
|
+
let lanesToResume: LaneInfo[] = [];
|
|
690
|
+
|
|
691
|
+
// Check if the lane argument is actually a tasks directory
|
|
692
|
+
if (options.lane && fs.existsSync(options.lane) && fs.statSync(options.lane).isDirectory()) {
|
|
693
|
+
const tasksDir = path.resolve(options.lane);
|
|
694
|
+
lanesToResume = allLanes.filter(l => l.needsResume && l.state?.tasksFile && path.resolve(l.state.tasksFile).startsWith(tasksDir));
|
|
705
695
|
|
|
706
|
-
if (
|
|
707
|
-
|
|
696
|
+
if (lanesToResume.length > 0) {
|
|
697
|
+
logger.info(`📂 Task directory detected: ${options.lane}`);
|
|
698
|
+
logger.info(`Resuming ${lanesToResume.length} lane(s) from this directory.`);
|
|
699
|
+
} else {
|
|
700
|
+
logger.warn(`No incomplete lanes found using tasks from directory: ${options.lane}`);
|
|
701
|
+
return;
|
|
708
702
|
}
|
|
709
|
-
|
|
703
|
+
} else if (options.all) {
|
|
704
|
+
lanesToResume = allLanes.filter(l => l.needsResume && l.state?.tasksFile);
|
|
705
|
+
} else if (options.lane) {
|
|
706
|
+
const lane = allLanes.find(l => l.name === options.lane);
|
|
707
|
+
if (!lane) {
|
|
708
|
+
throw new Error(`Lane '${options.lane}' not found in run directory.`);
|
|
709
|
+
}
|
|
710
|
+
if (!lane.needsResume) {
|
|
711
|
+
logger.success(`Lane '${options.lane}' is already completed.`);
|
|
712
|
+
return;
|
|
713
|
+
}
|
|
714
|
+
lanesToResume = [lane];
|
|
710
715
|
}
|
|
711
|
-
|
|
712
|
-
//
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
+
|
|
717
|
+
// Check for zombie lanes
|
|
718
|
+
const zombieCheck = checkAndFixZombieLanes(runDir);
|
|
719
|
+
if (zombieCheck.fixed.length > 0) {
|
|
720
|
+
logger.section('🔧 Zombie Lane Recovery');
|
|
721
|
+
logger.info(`Fixed ${zombieCheck.fixed.length} zombie lane(s): ${zombieCheck.fixed.join(', ')}`);
|
|
716
722
|
console.log('');
|
|
717
|
-
console.log('Usage: cursorflow resume <lane> [options]');
|
|
718
|
-
console.log(' cursorflow resume --all # Resume all incomplete lanes');
|
|
719
|
-
return;
|
|
720
723
|
}
|
|
721
724
|
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
if (!fs.existsSync(statePath)) {
|
|
726
|
-
throw new Error(`Lane state not found at ${statePath}. Is the lane name correct?`);
|
|
727
|
-
}
|
|
728
|
-
|
|
729
|
-
const state = loadState<LaneState>(statePath);
|
|
730
|
-
if (!state) {
|
|
731
|
-
throw new Error(`Failed to load state from ${statePath}`);
|
|
732
|
-
}
|
|
733
|
-
|
|
734
|
-
if (!state.tasksFile || !fs.existsSync(state.tasksFile)) {
|
|
735
|
-
throw new Error(`Original tasks file not found: ${state.tasksFile}. Resume impossible without task definition.`);
|
|
725
|
+
if (options.status) {
|
|
726
|
+
printAllLaneStatus(runDir);
|
|
727
|
+
return;
|
|
736
728
|
}
|
|
737
729
|
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
const report = runDoctor({
|
|
744
|
-
cwd: process.cwd(),
|
|
745
|
-
tasksDir,
|
|
746
|
-
includeCursorAgentChecks: false, // Skip agent checks for resume
|
|
747
|
-
});
|
|
748
|
-
|
|
749
|
-
// Only show blocking errors for resume
|
|
750
|
-
const blockingIssues = report.issues.filter(i =>
|
|
751
|
-
i.severity === 'error' &&
|
|
752
|
-
(i.id.startsWith('branch.') || i.id.startsWith('git.'))
|
|
753
|
-
);
|
|
754
|
-
|
|
755
|
-
if (blockingIssues.length > 0) {
|
|
756
|
-
logger.section('🛑 Pre-resume check found issues');
|
|
757
|
-
for (const issue of blockingIssues) {
|
|
758
|
-
logger.error(`${issue.title} (${issue.id})`, '❌');
|
|
759
|
-
console.log(` ${issue.message}`);
|
|
760
|
-
if (issue.details) console.log(` Details: ${issue.details}`);
|
|
761
|
-
if (issue.fixes?.length) {
|
|
762
|
-
console.log(' Fix:');
|
|
763
|
-
for (const fix of issue.fixes) console.log(` - ${fix}`);
|
|
764
|
-
}
|
|
765
|
-
console.log('');
|
|
766
|
-
}
|
|
767
|
-
throw new Error('Pre-resume checks failed. Use --skip-doctor to bypass (not recommended).');
|
|
768
|
-
}
|
|
769
|
-
|
|
770
|
-
// Show warnings but don't block
|
|
771
|
-
const warnings = report.issues.filter(i => i.severity === 'warn' && i.id.startsWith('branch.'));
|
|
772
|
-
if (warnings.length > 0) {
|
|
773
|
-
logger.warn(`${warnings.length} warning(s) found. Run 'cursorflow doctor' for details.`);
|
|
730
|
+
if (lanesToResume.length === 0) {
|
|
731
|
+
if (options.lane || options.all) {
|
|
732
|
+
logger.success('No lanes need to be resumed.');
|
|
733
|
+
} else {
|
|
734
|
+
printAllLaneStatus(runDir);
|
|
774
735
|
}
|
|
736
|
+
return;
|
|
775
737
|
}
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
logger.info(`Run: ${path.basename(runDir)}`);
|
|
779
|
-
logger.info(`Tasks: ${state.tasksFile}`);
|
|
780
|
-
logger.info(`Starting from task index: ${options.restart ? 0 : state.currentTaskIndex}`);
|
|
781
|
-
|
|
782
|
-
const { child } = spawnLaneResume(options.lane, laneDir, state, {
|
|
738
|
+
|
|
739
|
+
const result = await resumeLanes(lanesToResume, allLanes, {
|
|
783
740
|
restart: options.restart,
|
|
741
|
+
maxConcurrent: options.maxConcurrent,
|
|
742
|
+
skipDoctor: options.skipDoctor,
|
|
784
743
|
noGit: options.noGit,
|
|
785
744
|
executor: options.executor,
|
|
786
745
|
enhancedLogConfig: config.enhancedLogging,
|
|
787
746
|
});
|
|
788
747
|
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
child.on('exit', (code) => {
|
|
793
|
-
if (code === 0) {
|
|
794
|
-
logger.success(`Lane ${options.lane} completed successfully`);
|
|
795
|
-
resolve();
|
|
796
|
-
} else if (code === 2) {
|
|
797
|
-
logger.warn(`Lane ${options.lane} blocked on dependency change`);
|
|
798
|
-
resolve();
|
|
799
|
-
} else {
|
|
800
|
-
reject(new Error(`Lane ${options.lane} failed with exit code ${code}`));
|
|
801
|
-
}
|
|
802
|
-
});
|
|
803
|
-
|
|
804
|
-
child.on('error', (error) => {
|
|
805
|
-
reject(new Error(`Failed to start runner: ${error.message}`));
|
|
806
|
-
});
|
|
807
|
-
});
|
|
748
|
+
if (result.failed.length > 0) {
|
|
749
|
+
throw new Error(`${result.failed.length} lane(s) failed to complete`);
|
|
750
|
+
}
|
|
808
751
|
}
|
|
809
752
|
|
|
810
753
|
export = resume;
|