@litmers/cursorflow-orchestrator 0.1.18 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +25 -0
- package/README.md +25 -7
- package/commands/cursorflow-clean.md +19 -0
- package/commands/cursorflow-runs.md +59 -0
- package/commands/cursorflow-stop.md +55 -0
- package/dist/cli/clean.js +178 -6
- package/dist/cli/clean.js.map +1 -1
- package/dist/cli/index.js +12 -1
- package/dist/cli/index.js.map +1 -1
- package/dist/cli/init.js +8 -7
- package/dist/cli/init.js.map +1 -1
- package/dist/cli/logs.js +126 -77
- package/dist/cli/logs.js.map +1 -1
- package/dist/cli/monitor.d.ts +7 -0
- package/dist/cli/monitor.js +1021 -202
- package/dist/cli/monitor.js.map +1 -1
- package/dist/cli/prepare.js +39 -21
- package/dist/cli/prepare.js.map +1 -1
- package/dist/cli/resume.js +268 -163
- package/dist/cli/resume.js.map +1 -1
- package/dist/cli/run.js +11 -5
- package/dist/cli/run.js.map +1 -1
- package/dist/cli/runs.d.ts +5 -0
- package/dist/cli/runs.js +214 -0
- package/dist/cli/runs.js.map +1 -0
- package/dist/cli/setup-commands.js +0 -0
- package/dist/cli/signal.js +8 -8
- package/dist/cli/signal.js.map +1 -1
- package/dist/cli/stop.d.ts +5 -0
- package/dist/cli/stop.js +215 -0
- package/dist/cli/stop.js.map +1 -0
- package/dist/cli/tasks.d.ts +10 -0
- package/dist/cli/tasks.js +165 -0
- package/dist/cli/tasks.js.map +1 -0
- package/dist/core/auto-recovery.d.ts +212 -0
- package/dist/core/auto-recovery.js +737 -0
- package/dist/core/auto-recovery.js.map +1 -0
- package/dist/core/failure-policy.d.ts +156 -0
- package/dist/core/failure-policy.js +488 -0
- package/dist/core/failure-policy.js.map +1 -0
- package/dist/core/orchestrator.d.ts +16 -2
- package/dist/core/orchestrator.js +439 -105
- package/dist/core/orchestrator.js.map +1 -1
- package/dist/core/reviewer.d.ts +2 -0
- package/dist/core/reviewer.js +2 -0
- package/dist/core/reviewer.js.map +1 -1
- package/dist/core/runner.d.ts +33 -10
- package/dist/core/runner.js +374 -164
- package/dist/core/runner.js.map +1 -1
- package/dist/services/logging/buffer.d.ts +67 -0
- package/dist/services/logging/buffer.js +309 -0
- package/dist/services/logging/buffer.js.map +1 -0
- package/dist/services/logging/console.d.ts +89 -0
- package/dist/services/logging/console.js +169 -0
- package/dist/services/logging/console.js.map +1 -0
- package/dist/services/logging/file-writer.d.ts +71 -0
- package/dist/services/logging/file-writer.js +516 -0
- package/dist/services/logging/file-writer.js.map +1 -0
- package/dist/services/logging/formatter.d.ts +39 -0
- package/dist/services/logging/formatter.js +227 -0
- package/dist/services/logging/formatter.js.map +1 -0
- package/dist/services/logging/index.d.ts +11 -0
- package/dist/services/logging/index.js +30 -0
- package/dist/services/logging/index.js.map +1 -0
- package/dist/services/logging/parser.d.ts +31 -0
- package/dist/services/logging/parser.js +222 -0
- package/dist/services/logging/parser.js.map +1 -0
- package/dist/services/process/index.d.ts +59 -0
- package/dist/services/process/index.js +257 -0
- package/dist/services/process/index.js.map +1 -0
- package/dist/types/agent.d.ts +20 -0
- package/dist/types/agent.js +6 -0
- package/dist/types/agent.js.map +1 -0
- package/dist/types/config.d.ts +65 -0
- package/dist/types/config.js +6 -0
- package/dist/types/config.js.map +1 -0
- package/dist/types/events.d.ts +125 -0
- package/dist/types/events.js +6 -0
- package/dist/types/events.js.map +1 -0
- package/dist/types/index.d.ts +12 -0
- package/dist/types/index.js +37 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/lane.d.ts +43 -0
- package/dist/types/lane.js +6 -0
- package/dist/types/lane.js.map +1 -0
- package/dist/types/logging.d.ts +71 -0
- package/dist/types/logging.js +16 -0
- package/dist/types/logging.js.map +1 -0
- package/dist/types/review.d.ts +17 -0
- package/dist/types/review.js +6 -0
- package/dist/types/review.js.map +1 -0
- package/dist/types/run.d.ts +32 -0
- package/dist/types/run.js +6 -0
- package/dist/types/run.js.map +1 -0
- package/dist/types/task.d.ts +71 -0
- package/dist/types/task.js +6 -0
- package/dist/types/task.js.map +1 -0
- package/dist/ui/components.d.ts +134 -0
- package/dist/ui/components.js +389 -0
- package/dist/ui/components.js.map +1 -0
- package/dist/ui/log-viewer.d.ts +49 -0
- package/dist/ui/log-viewer.js +449 -0
- package/dist/ui/log-viewer.js.map +1 -0
- package/dist/utils/checkpoint.d.ts +87 -0
- package/dist/utils/checkpoint.js +317 -0
- package/dist/utils/checkpoint.js.map +1 -0
- package/dist/utils/config.d.ts +4 -0
- package/dist/utils/config.js +18 -8
- package/dist/utils/config.js.map +1 -1
- package/dist/utils/cursor-agent.js.map +1 -1
- package/dist/utils/dependency.d.ts +74 -0
- package/dist/utils/dependency.js +420 -0
- package/dist/utils/dependency.js.map +1 -0
- package/dist/utils/doctor.js +17 -11
- package/dist/utils/doctor.js.map +1 -1
- package/dist/utils/enhanced-logger.d.ts +10 -33
- package/dist/utils/enhanced-logger.js +108 -20
- package/dist/utils/enhanced-logger.js.map +1 -1
- package/dist/utils/git.d.ts +121 -0
- package/dist/utils/git.js +484 -11
- package/dist/utils/git.js.map +1 -1
- package/dist/utils/health.d.ts +91 -0
- package/dist/utils/health.js +556 -0
- package/dist/utils/health.js.map +1 -0
- package/dist/utils/lock.d.ts +95 -0
- package/dist/utils/lock.js +332 -0
- package/dist/utils/lock.js.map +1 -0
- package/dist/utils/log-buffer.d.ts +17 -0
- package/dist/utils/log-buffer.js +14 -0
- package/dist/utils/log-buffer.js.map +1 -0
- package/dist/utils/log-constants.d.ts +23 -0
- package/dist/utils/log-constants.js +28 -0
- package/dist/utils/log-constants.js.map +1 -0
- package/dist/utils/log-formatter.d.ts +25 -0
- package/dist/utils/log-formatter.js +237 -0
- package/dist/utils/log-formatter.js.map +1 -0
- package/dist/utils/log-service.d.ts +19 -0
- package/dist/utils/log-service.js +47 -0
- package/dist/utils/log-service.js.map +1 -0
- package/dist/utils/logger.d.ts +46 -27
- package/dist/utils/logger.js +82 -60
- package/dist/utils/logger.js.map +1 -1
- package/dist/utils/path.d.ts +19 -0
- package/dist/utils/path.js +77 -0
- package/dist/utils/path.js.map +1 -0
- package/dist/utils/process-manager.d.ts +21 -0
- package/dist/utils/process-manager.js +138 -0
- package/dist/utils/process-manager.js.map +1 -0
- package/dist/utils/retry.d.ts +121 -0
- package/dist/utils/retry.js +374 -0
- package/dist/utils/retry.js.map +1 -0
- package/dist/utils/run-service.d.ts +88 -0
- package/dist/utils/run-service.js +412 -0
- package/dist/utils/run-service.js.map +1 -0
- package/dist/utils/state.d.ts +62 -3
- package/dist/utils/state.js +317 -11
- package/dist/utils/state.js.map +1 -1
- package/dist/utils/task-service.d.ts +82 -0
- package/dist/utils/task-service.js +348 -0
- package/dist/utils/task-service.js.map +1 -0
- package/dist/utils/template.d.ts +14 -0
- package/dist/utils/template.js +122 -0
- package/dist/utils/template.js.map +1 -0
- package/dist/utils/types.d.ts +2 -271
- package/dist/utils/types.js +16 -0
- package/dist/utils/types.js.map +1 -1
- package/package.json +38 -23
- package/scripts/ai-security-check.js +0 -1
- package/scripts/local-security-gate.sh +0 -0
- package/scripts/monitor-lanes.sh +94 -0
- package/scripts/patches/test-cursor-agent.js +0 -1
- package/scripts/release.sh +0 -0
- package/scripts/setup-security.sh +0 -0
- package/scripts/stream-logs.sh +72 -0
- package/scripts/verify-and-fix.sh +0 -0
- package/src/cli/clean.ts +187 -6
- package/src/cli/index.ts +12 -1
- package/src/cli/init.ts +8 -7
- package/src/cli/logs.ts +124 -77
- package/src/cli/monitor.ts +1815 -898
- package/src/cli/prepare.ts +41 -21
- package/src/cli/resume.ts +753 -626
- package/src/cli/run.ts +12 -5
- package/src/cli/runs.ts +212 -0
- package/src/cli/setup-commands.ts +0 -0
- package/src/cli/signal.ts +8 -7
- package/src/cli/stop.ts +209 -0
- package/src/cli/tasks.ts +154 -0
- package/src/core/auto-recovery.ts +909 -0
- package/src/core/failure-policy.ts +592 -0
- package/src/core/orchestrator.ts +1131 -704
- package/src/core/reviewer.ts +4 -0
- package/src/core/runner.ts +444 -180
- package/src/services/logging/buffer.ts +326 -0
- package/src/services/logging/console.ts +193 -0
- package/src/services/logging/file-writer.ts +526 -0
- package/src/services/logging/formatter.ts +268 -0
- package/src/services/logging/index.ts +16 -0
- package/src/services/logging/parser.ts +232 -0
- package/src/services/process/index.ts +261 -0
- package/src/types/agent.ts +24 -0
- package/src/types/config.ts +79 -0
- package/src/types/events.ts +156 -0
- package/src/types/index.ts +29 -0
- package/src/types/lane.ts +56 -0
- package/src/types/logging.ts +96 -0
- package/src/types/review.ts +20 -0
- package/src/types/run.ts +37 -0
- package/src/types/task.ts +79 -0
- package/src/ui/components.ts +430 -0
- package/src/ui/log-viewer.ts +485 -0
- package/src/utils/checkpoint.ts +374 -0
- package/src/utils/config.ts +18 -8
- package/src/utils/cursor-agent.ts +1 -1
- package/src/utils/dependency.ts +482 -0
- package/src/utils/doctor.ts +18 -11
- package/src/utils/enhanced-logger.ts +122 -60
- package/src/utils/git.ts +517 -11
- package/src/utils/health.ts +596 -0
- package/src/utils/lock.ts +346 -0
- package/src/utils/log-buffer.ts +28 -0
- package/src/utils/log-constants.ts +26 -0
- package/src/utils/log-formatter.ts +245 -0
- package/src/utils/log-service.ts +49 -0
- package/src/utils/logger.ts +100 -51
- package/src/utils/path.ts +45 -0
- package/src/utils/process-manager.ts +100 -0
- package/src/utils/retry.ts +413 -0
- package/src/utils/run-service.ts +433 -0
- package/src/utils/state.ts +385 -11
- package/src/utils/task-service.ts +370 -0
- package/src/utils/template.ts +92 -0
- package/src/utils/types.ts +2 -314
- package/templates/basic.json +21 -0
package/dist/cli/resume.js
CHANGED
|
@@ -42,34 +42,40 @@ const logger = __importStar(require("../utils/logger"));
|
|
|
42
42
|
const config_1 = require("../utils/config");
|
|
43
43
|
const state_1 = require("../utils/state");
|
|
44
44
|
const doctor_1 = require("../utils/doctor");
|
|
45
|
+
const path_1 = require("../utils/path");
|
|
46
|
+
const enhanced_logger_1 = require("../utils/enhanced-logger");
|
|
47
|
+
const log_formatter_1 = require("../utils/log-formatter");
|
|
45
48
|
function printHelp() {
|
|
46
|
-
console.log(`
|
|
47
|
-
Usage: cursorflow resume [lane] [options]
|
|
48
|
-
|
|
49
|
-
Resume interrupted or failed lanes.
|
|
50
|
-
|
|
51
|
-
Options:
|
|
52
|
-
<lane> Lane name
|
|
53
|
-
--all Resume ALL incomplete/failed lanes
|
|
54
|
-
--status Show status of all lanes in the run (no resume)
|
|
55
|
-
--run-dir <path> Use a specific run directory (default: latest)
|
|
56
|
-
--max-concurrent <n> Max lanes to run in parallel (default: 3)
|
|
57
|
-
--clean Clean up existing worktree before resuming
|
|
58
|
-
--restart Restart from the first task (index 0)
|
|
59
|
-
--skip-doctor Skip environment/branch checks (not recommended)
|
|
60
|
-
--
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
cursorflow resume
|
|
66
|
-
cursorflow resume --all
|
|
67
|
-
cursorflow resume
|
|
49
|
+
console.log(`
|
|
50
|
+
Usage: cursorflow resume [lane] [options]
|
|
51
|
+
|
|
52
|
+
Resume interrupted or failed lanes.
|
|
53
|
+
|
|
54
|
+
Options:
|
|
55
|
+
<lane> Lane name or tasks directory to resume
|
|
56
|
+
--all Resume ALL incomplete/failed lanes
|
|
57
|
+
--status Show status of all lanes in the run (no resume)
|
|
58
|
+
--run-dir <path> Use a specific run directory (default: latest)
|
|
59
|
+
--max-concurrent <n> Max lanes to run in parallel (default: 3)
|
|
60
|
+
--clean Clean up existing worktree before resuming
|
|
61
|
+
--restart Restart from the first task (index 0)
|
|
62
|
+
--skip-doctor Skip environment/branch checks (not recommended)
|
|
63
|
+
--no-git Disable Git operations (must match original run)
|
|
64
|
+
--executor <type> Override executor (default: cursor-agent)
|
|
65
|
+
--help, -h Show help
|
|
66
|
+
|
|
67
|
+
Examples:
|
|
68
|
+
cursorflow resume --status # Check status of all lanes
|
|
69
|
+
cursorflow resume --all # Resume all incomplete lanes
|
|
70
|
+
cursorflow resume lane-1 # Resume single lane
|
|
71
|
+
cursorflow resume _cursorflow/tasks/feat1 # Resume all lanes in directory
|
|
72
|
+
cursorflow resume --all --restart # Restart all incomplete lanes from task 0
|
|
68
73
|
`);
|
|
69
74
|
}
|
|
70
75
|
function parseArgs(args) {
|
|
71
76
|
const runDirIdx = args.indexOf('--run-dir');
|
|
72
77
|
const maxConcurrentIdx = args.indexOf('--max-concurrent');
|
|
78
|
+
const executorIdx = args.indexOf('--executor');
|
|
73
79
|
return {
|
|
74
80
|
lane: args.find(a => !a.startsWith('--')) || null,
|
|
75
81
|
runDir: runDirIdx >= 0 ? args[runDirIdx + 1] || null : null,
|
|
@@ -80,20 +86,22 @@ function parseArgs(args) {
|
|
|
80
86
|
status: args.includes('--status'),
|
|
81
87
|
maxConcurrent: maxConcurrentIdx >= 0 ? parseInt(args[maxConcurrentIdx + 1] || '3') : 3,
|
|
82
88
|
help: args.includes('--help') || args.includes('-h'),
|
|
89
|
+
noGit: args.includes('--no-git'),
|
|
90
|
+
executor: executorIdx >= 0 ? args[executorIdx + 1] || null : null,
|
|
83
91
|
};
|
|
84
92
|
}
|
|
85
93
|
/**
|
|
86
94
|
* Find the latest run directory
|
|
87
95
|
*/
|
|
88
96
|
function findLatestRunDir(logsDir) {
|
|
89
|
-
const runsDir =
|
|
97
|
+
const runsDir = (0, path_1.safeJoin)(logsDir, 'runs');
|
|
90
98
|
if (!fs.existsSync(runsDir))
|
|
91
99
|
return null;
|
|
92
100
|
const runs = fs.readdirSync(runsDir)
|
|
93
101
|
.filter(d => d.startsWith('run-'))
|
|
94
102
|
.sort()
|
|
95
103
|
.reverse();
|
|
96
|
-
return runs.length > 0 ?
|
|
104
|
+
return runs.length > 0 ? (0, path_1.safeJoin)(runsDir, runs[0]) : null;
|
|
97
105
|
}
|
|
98
106
|
/**
|
|
99
107
|
* Status indicator colors
|
|
@@ -109,25 +117,156 @@ const STATUS_COLORS = {
|
|
|
109
117
|
unknown: '\x1b[90m', // gray
|
|
110
118
|
};
|
|
111
119
|
const RESET = '\x1b[0m';
|
|
120
|
+
/**
|
|
121
|
+
* Check if a process is alive by its PID
|
|
122
|
+
*/
|
|
123
|
+
function isProcessAlive(pid) {
|
|
124
|
+
try {
|
|
125
|
+
// On Unix-like systems, sending signal 0 checks if process exists
|
|
126
|
+
process.kill(pid, 0);
|
|
127
|
+
return true;
|
|
128
|
+
}
|
|
129
|
+
catch {
|
|
130
|
+
return false;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Check for zombie "running" lanes and fix them
|
|
135
|
+
* A zombie lane is one that has status "running" but its process is dead
|
|
136
|
+
*/
|
|
137
|
+
function checkAndFixZombieLanes(runDir) {
|
|
138
|
+
const lanesDir = (0, path_1.safeJoin)(runDir, 'lanes');
|
|
139
|
+
if (!fs.existsSync(lanesDir)) {
|
|
140
|
+
return { fixed: [], pofCreated: false };
|
|
141
|
+
}
|
|
142
|
+
const fixed = [];
|
|
143
|
+
const zombieDetails = [];
|
|
144
|
+
const laneDirs = fs.readdirSync(lanesDir)
|
|
145
|
+
.filter(f => fs.statSync((0, path_1.safeJoin)(lanesDir, f)).isDirectory());
|
|
146
|
+
for (const laneName of laneDirs) {
|
|
147
|
+
const dir = (0, path_1.safeJoin)(lanesDir, laneName);
|
|
148
|
+
const statePath = (0, path_1.safeJoin)(dir, 'state.json');
|
|
149
|
+
if (!fs.existsSync(statePath))
|
|
150
|
+
continue;
|
|
151
|
+
const state = (0, state_1.loadState)(statePath);
|
|
152
|
+
if (!state)
|
|
153
|
+
continue;
|
|
154
|
+
// Check for zombie: status is "running" but process is dead
|
|
155
|
+
if (state.status === 'running' && state.pid) {
|
|
156
|
+
const alive = isProcessAlive(state.pid);
|
|
157
|
+
if (!alive) {
|
|
158
|
+
logger.warn(`🧟 Zombie lane detected: ${laneName} (PID ${state.pid} is dead)`);
|
|
159
|
+
// Update state to failed
|
|
160
|
+
const updatedState = {
|
|
161
|
+
...state,
|
|
162
|
+
status: 'failed',
|
|
163
|
+
error: `Process terminated unexpectedly (PID ${state.pid} was running but is now dead)`,
|
|
164
|
+
endTime: Date.now(),
|
|
165
|
+
};
|
|
166
|
+
(0, state_1.saveState)(statePath, updatedState);
|
|
167
|
+
fixed.push(laneName);
|
|
168
|
+
zombieDetails.push({
|
|
169
|
+
name: laneName,
|
|
170
|
+
pid: state.pid,
|
|
171
|
+
taskIndex: state.currentTaskIndex,
|
|
172
|
+
totalTasks: state.totalTasks,
|
|
173
|
+
});
|
|
174
|
+
logger.info(` → Status changed to 'failed', ready for resume`);
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
// Create POF file if any zombies were found
|
|
179
|
+
let pofCreated = false;
|
|
180
|
+
if (zombieDetails.length > 0) {
|
|
181
|
+
const config = (0, config_1.loadConfig)();
|
|
182
|
+
const pofDir = (0, config_1.getPofDir)(config);
|
|
183
|
+
if (!fs.existsSync(pofDir)) {
|
|
184
|
+
fs.mkdirSync(pofDir, { recursive: true });
|
|
185
|
+
}
|
|
186
|
+
const runId = path.basename(runDir);
|
|
187
|
+
const pofPath = (0, path_1.safeJoin)(pofDir, `pof-${runId}.json`);
|
|
188
|
+
let existingPof = null;
|
|
189
|
+
try {
|
|
190
|
+
existingPof = JSON.parse(fs.readFileSync(pofPath, 'utf-8'));
|
|
191
|
+
}
|
|
192
|
+
catch {
|
|
193
|
+
// Ignore errors (file might not exist)
|
|
194
|
+
}
|
|
195
|
+
const pof = {
|
|
196
|
+
title: 'Run Failure Post-mortem',
|
|
197
|
+
runId: path.basename(runDir),
|
|
198
|
+
failureTime: new Date().toISOString(),
|
|
199
|
+
detectedAt: new Date().toISOString(),
|
|
200
|
+
summary: `${zombieDetails.length} lane(s) found with dead processes (zombie state)`,
|
|
201
|
+
rootCause: {
|
|
202
|
+
type: 'ZOMBIE_PROCESS',
|
|
203
|
+
description: 'Lane processes were marked as running but the processes are no longer alive',
|
|
204
|
+
symptoms: [
|
|
205
|
+
'Process PIDs no longer exist in the system',
|
|
206
|
+
'Lanes were stuck in "running" state',
|
|
207
|
+
'No completion or error was recorded before process death',
|
|
208
|
+
],
|
|
209
|
+
},
|
|
210
|
+
affectedLanes: zombieDetails.map(z => ({
|
|
211
|
+
name: z.name,
|
|
212
|
+
status: 'failed (was: running)',
|
|
213
|
+
task: `[${z.taskIndex + 1}/${z.totalTasks}]`,
|
|
214
|
+
taskIndex: z.taskIndex,
|
|
215
|
+
pid: z.pid,
|
|
216
|
+
reason: 'Process terminated unexpectedly',
|
|
217
|
+
})),
|
|
218
|
+
possibleCauses: [
|
|
219
|
+
'System killed process due to memory pressure (OOM)',
|
|
220
|
+
'User killed process manually (Ctrl+C, kill command)',
|
|
221
|
+
'Agent timeout exceeded and process was terminated',
|
|
222
|
+
'System restart or crash',
|
|
223
|
+
'Agent hung and watchdog terminated it',
|
|
224
|
+
],
|
|
225
|
+
recovery: {
|
|
226
|
+
command: `cursorflow resume --all --run-dir ${runDir}`,
|
|
227
|
+
description: 'Resume all failed lanes from their last checkpoint',
|
|
228
|
+
alternativeCommand: `cursorflow resume --all --restart --run-dir ${runDir}`,
|
|
229
|
+
alternativeDescription: 'Restart all failed lanes from the beginning',
|
|
230
|
+
},
|
|
231
|
+
// Merge with existing POF if present
|
|
232
|
+
previousFailures: existingPof ? [existingPof] : undefined,
|
|
233
|
+
};
|
|
234
|
+
// Use atomic write: write to temp file then rename
|
|
235
|
+
const tempPath = `${pofPath}.${Math.random().toString(36).substring(2, 7)}.tmp`;
|
|
236
|
+
try {
|
|
237
|
+
fs.writeFileSync(tempPath, JSON.stringify(pof, null, 2), 'utf8');
|
|
238
|
+
fs.renameSync(tempPath, pofPath);
|
|
239
|
+
pofCreated = true;
|
|
240
|
+
logger.info(`📋 POF file created: ${pofPath}`);
|
|
241
|
+
}
|
|
242
|
+
catch (err) {
|
|
243
|
+
// If temp file was created, try to clean it up
|
|
244
|
+
try {
|
|
245
|
+
if (fs.existsSync(tempPath))
|
|
246
|
+
fs.unlinkSync(tempPath);
|
|
247
|
+
}
|
|
248
|
+
catch { /* ignore */ }
|
|
249
|
+
throw err;
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
return { fixed, pofCreated };
|
|
253
|
+
}
|
|
112
254
|
/**
|
|
113
255
|
* Get all lane statuses from a run directory
|
|
114
256
|
*/
|
|
115
257
|
function getAllLaneStatuses(runDir) {
|
|
116
|
-
const lanesDir =
|
|
258
|
+
const lanesDir = (0, path_1.safeJoin)(runDir, 'lanes');
|
|
117
259
|
if (!fs.existsSync(lanesDir)) {
|
|
118
260
|
return [];
|
|
119
261
|
}
|
|
120
262
|
const lanes = fs.readdirSync(lanesDir)
|
|
121
|
-
.filter(f => fs.statSync(
|
|
263
|
+
.filter(f => fs.statSync((0, path_1.safeJoin)(lanesDir, f)).isDirectory())
|
|
122
264
|
.map(name => {
|
|
123
|
-
const dir =
|
|
124
|
-
const statePath =
|
|
265
|
+
const dir = (0, path_1.safeJoin)(lanesDir, name);
|
|
266
|
+
const statePath = (0, path_1.safeJoin)(dir, 'state.json');
|
|
125
267
|
const state = fs.existsSync(statePath) ? (0, state_1.loadState)(statePath) : null;
|
|
126
|
-
// Determine if lane needs resume
|
|
127
|
-
const needsResume = state ? (state.status
|
|
128
|
-
state.status === 'paused' ||
|
|
129
|
-
state.status === 'running' || // If process crashed mid-run
|
|
130
|
-
(state.status === 'pending' && state.currentTaskIndex > 0)) : false;
|
|
268
|
+
// Determine if lane needs resume: everything that is not completed
|
|
269
|
+
const needsResume = state ? (state.status !== 'completed') : true;
|
|
131
270
|
const isCompleted = state?.status === 'completed';
|
|
132
271
|
const dependsOn = state?.dependsOn || [];
|
|
133
272
|
return { name, dir, state, needsResume, dependsOn, isCompleted };
|
|
@@ -229,7 +368,7 @@ function printAllLaneStatus(runDir) {
|
|
|
229
368
|
return { total: lanes.length, completed: completedCount, needsResume: needsResumeCount };
|
|
230
369
|
}
|
|
231
370
|
/**
|
|
232
|
-
* Resume a single lane and return the child process
|
|
371
|
+
* Resume a single lane and return the child process and its log manager
|
|
233
372
|
*/
|
|
234
373
|
function spawnLaneResume(laneName, laneDir, state, options) {
|
|
235
374
|
const runnerPath = require.resolve('../core/runner');
|
|
@@ -240,11 +379,46 @@ function spawnLaneResume(laneName, laneDir, state, options) {
|
|
|
240
379
|
'--run-dir', laneDir,
|
|
241
380
|
'--start-index', String(startIndex),
|
|
242
381
|
];
|
|
382
|
+
if (state.worktreeDir) {
|
|
383
|
+
runnerArgs.push('--worktree-dir', state.worktreeDir);
|
|
384
|
+
}
|
|
385
|
+
if (options.noGit) {
|
|
386
|
+
runnerArgs.push('--no-git');
|
|
387
|
+
}
|
|
388
|
+
// Explicitly pass pipeline branch if available (either from state or override)
|
|
389
|
+
const branch = options.pipelineBranch || state.pipelineBranch;
|
|
390
|
+
if (branch) {
|
|
391
|
+
runnerArgs.push('--pipeline-branch', branch);
|
|
392
|
+
}
|
|
393
|
+
// Pass executor if provided
|
|
394
|
+
if (options.executor) {
|
|
395
|
+
runnerArgs.push('--executor', options.executor);
|
|
396
|
+
}
|
|
397
|
+
const logManager = (0, enhanced_logger_1.createLogManager)(laneDir, laneName, options.enhancedLogConfig || {}, (msg) => {
|
|
398
|
+
const formatted = (0, log_formatter_1.formatMessageForConsole)(msg, {
|
|
399
|
+
laneLabel: `[${laneName}]`,
|
|
400
|
+
includeTimestamp: true
|
|
401
|
+
});
|
|
402
|
+
process.stdout.write(formatted + '\n');
|
|
403
|
+
});
|
|
243
404
|
const child = (0, child_process_1.spawn)('node', runnerArgs, {
|
|
244
|
-
stdio: '
|
|
405
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
245
406
|
env: process.env,
|
|
246
407
|
});
|
|
247
|
-
|
|
408
|
+
if (child.stdout) {
|
|
409
|
+
child.stdout.on('data', (data) => {
|
|
410
|
+
logManager.writeStdout(data);
|
|
411
|
+
});
|
|
412
|
+
}
|
|
413
|
+
if (child.stderr) {
|
|
414
|
+
child.stderr.on('data', (data) => {
|
|
415
|
+
logManager.writeStderr(data);
|
|
416
|
+
});
|
|
417
|
+
}
|
|
418
|
+
child.on('exit', () => {
|
|
419
|
+
logManager.close();
|
|
420
|
+
});
|
|
421
|
+
return { child, logManager };
|
|
248
422
|
}
|
|
249
423
|
/**
|
|
250
424
|
* Wait for a child process to exit
|
|
@@ -262,14 +436,7 @@ function waitForChild(child) {
|
|
|
262
436
|
/**
|
|
263
437
|
* Resume multiple lanes with concurrency control and dependency awareness
|
|
264
438
|
*/
|
|
265
|
-
async function
|
|
266
|
-
const allLanes = getAllLaneStatuses(runDir);
|
|
267
|
-
const lanesToResume = allLanes.filter(l => l.needsResume && l.state?.tasksFile);
|
|
268
|
-
if (lanesToResume.length === 0) {
|
|
269
|
-
logger.success('All lanes are already completed! Nothing to resume.');
|
|
270
|
-
return { succeeded: [], failed: [], skipped: [] };
|
|
271
|
-
}
|
|
272
|
-
// Check for lanes with unmet dependencies that can never be satisfied
|
|
439
|
+
async function resumeLanes(lanesToResume, allLanes, options) {
|
|
273
440
|
const completedSet = new Set(allLanes.filter(l => l.isCompleted).map(l => l.name));
|
|
274
441
|
const toResumeNames = new Set(lanesToResume.map(l => l.name));
|
|
275
442
|
const skippedLanes = [];
|
|
@@ -292,16 +459,9 @@ async function resumeAllLanes(runDir, options) {
|
|
|
292
459
|
logger.section(`🔁 Resuming ${resolvableLanes.length} Lane(s)`);
|
|
293
460
|
logger.info(`Max concurrent: ${options.maxConcurrent}`);
|
|
294
461
|
logger.info(`Mode: ${options.restart ? 'Restart from beginning' : 'Continue from last task'}`);
|
|
295
|
-
// Show dependency order
|
|
296
|
-
const lanesWithDeps = resolvableLanes.filter(l => l.dependsOn.length > 0);
|
|
297
|
-
if (lanesWithDeps.length > 0) {
|
|
298
|
-
logger.info(`Dependency-aware: ${lanesWithDeps.length} lane(s) have dependencies`);
|
|
299
|
-
}
|
|
300
|
-
console.log('');
|
|
301
462
|
// Run doctor check once if needed (check git status)
|
|
302
463
|
if (!options.skipDoctor) {
|
|
303
464
|
logger.info('Running pre-flight checks...');
|
|
304
|
-
// Use the first lane's tasksDir for doctor check
|
|
305
465
|
const firstLane = resolvableLanes[0];
|
|
306
466
|
const tasksDir = path.dirname(firstLane.state.tasksFile);
|
|
307
467
|
const report = (0, doctor_1.runDoctor)({
|
|
@@ -322,15 +482,10 @@ async function resumeAllLanes(runDir, options) {
|
|
|
322
482
|
}
|
|
323
483
|
const succeeded = [];
|
|
324
484
|
const failed = [];
|
|
325
|
-
// Create a mutable set for tracking completed lanes (including those from this session)
|
|
326
485
|
const sessionCompleted = new Set(completedSet);
|
|
327
|
-
// Queue management with dependency awareness
|
|
328
486
|
const pending = new Set(resolvableLanes.map(l => l.name));
|
|
329
487
|
const active = new Map();
|
|
330
488
|
const laneMap = new Map(resolvableLanes.map(l => [l.name, l]));
|
|
331
|
-
/**
|
|
332
|
-
* Find the next lane that can be started (all dependencies met)
|
|
333
|
-
*/
|
|
334
489
|
const findReadyLane = () => {
|
|
335
490
|
for (const laneName of pending) {
|
|
336
491
|
const lane = laneMap.get(laneName);
|
|
@@ -340,21 +495,15 @@ async function resumeAllLanes(runDir, options) {
|
|
|
340
495
|
}
|
|
341
496
|
return null;
|
|
342
497
|
};
|
|
343
|
-
/**
|
|
344
|
-
* Process lanes with dependency awareness
|
|
345
|
-
*/
|
|
346
498
|
const processNext = () => {
|
|
347
499
|
while (active.size < options.maxConcurrent) {
|
|
348
500
|
const lane = findReadyLane();
|
|
349
501
|
if (!lane) {
|
|
350
|
-
// No lane ready to start
|
|
351
502
|
if (pending.size > 0 && active.size === 0) {
|
|
352
|
-
// Deadlock: pending lanes exist but none can start and none are running
|
|
353
503
|
const pendingList = Array.from(pending).join(', ');
|
|
354
504
|
logger.error(`Deadlock detected! Lanes waiting: ${pendingList}`);
|
|
355
|
-
for (const ln of pending)
|
|
505
|
+
for (const ln of pending)
|
|
356
506
|
failed.push(ln);
|
|
357
|
-
}
|
|
358
507
|
pending.clear();
|
|
359
508
|
}
|
|
360
509
|
break;
|
|
@@ -362,17 +511,19 @@ async function resumeAllLanes(runDir, options) {
|
|
|
362
511
|
pending.delete(lane.name);
|
|
363
512
|
const depsInfo = lane.dependsOn.length > 0 ? ` (after: ${lane.dependsOn.join(', ')})` : '';
|
|
364
513
|
logger.info(`Starting: ${lane.name} (task ${lane.state.currentTaskIndex}/${lane.state.totalTasks})${depsInfo}`);
|
|
365
|
-
const child = spawnLaneResume(lane.name, lane.dir, lane.state, {
|
|
514
|
+
const { child } = spawnLaneResume(lane.name, lane.dir, lane.state, {
|
|
366
515
|
restart: options.restart,
|
|
516
|
+
noGit: options.noGit,
|
|
517
|
+
executor: options.executor,
|
|
518
|
+
enhancedLogConfig: options.enhancedLogConfig,
|
|
367
519
|
});
|
|
368
520
|
active.set(lane.name, child);
|
|
369
|
-
// Handle completion
|
|
370
521
|
waitForChild(child).then(code => {
|
|
371
522
|
active.delete(lane.name);
|
|
372
523
|
if (code === 0) {
|
|
373
524
|
logger.success(`✓ ${lane.name} completed`);
|
|
374
525
|
succeeded.push(lane.name);
|
|
375
|
-
sessionCompleted.add(lane.name);
|
|
526
|
+
sessionCompleted.add(lane.name);
|
|
376
527
|
}
|
|
377
528
|
else if (code === 2) {
|
|
378
529
|
logger.warn(`⚠ ${lane.name} blocked on dependency change`);
|
|
@@ -382,7 +533,6 @@ async function resumeAllLanes(runDir, options) {
|
|
|
382
533
|
logger.error(`✗ ${lane.name} failed (exit ${code})`);
|
|
383
534
|
failed.push(lane.name);
|
|
384
535
|
}
|
|
385
|
-
// Try to start more lanes now that one completed
|
|
386
536
|
processNext();
|
|
387
537
|
}).catch(err => {
|
|
388
538
|
active.delete(lane.name);
|
|
@@ -392,26 +542,20 @@ async function resumeAllLanes(runDir, options) {
|
|
|
392
542
|
});
|
|
393
543
|
}
|
|
394
544
|
};
|
|
395
|
-
// Start initial batch
|
|
396
545
|
processNext();
|
|
397
|
-
// Wait for all to complete
|
|
398
546
|
while (active.size > 0 || pending.size > 0) {
|
|
399
547
|
await new Promise(resolve => setTimeout(resolve, 1000));
|
|
400
|
-
// Check if we can start more (in case completion handlers haven't triggered processNext yet)
|
|
401
548
|
if (active.size < options.maxConcurrent && pending.size > 0) {
|
|
402
549
|
processNext();
|
|
403
550
|
}
|
|
404
551
|
}
|
|
405
|
-
// Summary
|
|
406
552
|
console.log('');
|
|
407
553
|
logger.section('📊 Resume Summary');
|
|
408
554
|
logger.info(`Succeeded: ${succeeded.length}`);
|
|
409
|
-
if (failed.length > 0)
|
|
555
|
+
if (failed.length > 0)
|
|
410
556
|
logger.error(`Failed: ${failed.length} (${failed.join(', ')})`);
|
|
411
|
-
|
|
412
|
-
if (skippedLanes.length > 0) {
|
|
557
|
+
if (skippedLanes.length > 0)
|
|
413
558
|
logger.warn(`Skipped: ${skippedLanes.length} (${skippedLanes.join(', ')})`);
|
|
414
|
-
}
|
|
415
559
|
return { succeeded, failed, skipped: skippedLanes };
|
|
416
560
|
}
|
|
417
561
|
async function resume(args) {
|
|
@@ -422,7 +566,6 @@ async function resume(args) {
|
|
|
422
566
|
}
|
|
423
567
|
const config = (0, config_1.loadConfig)();
|
|
424
568
|
const logsDir = (0, config_1.getLogsDir)(config);
|
|
425
|
-
// Find run directory
|
|
426
569
|
let runDir = options.runDir;
|
|
427
570
|
if (!runDir) {
|
|
428
571
|
runDir = findLatestRunDir(logsDir);
|
|
@@ -430,104 +573,66 @@ async function resume(args) {
|
|
|
430
573
|
if (!runDir || !fs.existsSync(runDir)) {
|
|
431
574
|
throw new Error(`Run directory not found: ${runDir || 'latest'}. Have you run any tasks yet?`);
|
|
432
575
|
}
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
throw new Error(`${result.failed.length} lane(s) failed to complete`);
|
|
576
|
+
const allLanes = getAllLaneStatuses(runDir);
|
|
577
|
+
let lanesToResume = [];
|
|
578
|
+
// Check if the lane argument is actually a tasks directory
|
|
579
|
+
if (options.lane && fs.existsSync(options.lane) && fs.statSync(options.lane).isDirectory()) {
|
|
580
|
+
const tasksDir = path.resolve(options.lane);
|
|
581
|
+
lanesToResume = allLanes.filter(l => l.needsResume && l.state?.tasksFile && path.resolve(l.state.tasksFile).startsWith(tasksDir));
|
|
582
|
+
if (lanesToResume.length > 0) {
|
|
583
|
+
logger.info(`📂 Task directory detected: ${options.lane}`);
|
|
584
|
+
logger.info(`Resuming ${lanesToResume.length} lane(s) from this directory.`);
|
|
585
|
+
}
|
|
586
|
+
else {
|
|
587
|
+
logger.warn(`No incomplete lanes found using tasks from directory: ${options.lane}`);
|
|
588
|
+
return;
|
|
447
589
|
}
|
|
448
|
-
return;
|
|
449
590
|
}
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
// Show status by default if no lane specified
|
|
453
|
-
printAllLaneStatus(runDir);
|
|
454
|
-
console.log('');
|
|
455
|
-
console.log('Usage: cursorflow resume <lane> [options]');
|
|
456
|
-
console.log(' cursorflow resume --all # Resume all incomplete lanes');
|
|
457
|
-
return;
|
|
591
|
+
else if (options.all) {
|
|
592
|
+
lanesToResume = allLanes.filter(l => l.needsResume && l.state?.tasksFile);
|
|
458
593
|
}
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
594
|
+
else if (options.lane) {
|
|
595
|
+
const lane = allLanes.find(l => l.name === options.lane);
|
|
596
|
+
if (!lane) {
|
|
597
|
+
throw new Error(`Lane '${options.lane}' not found in run directory.`);
|
|
598
|
+
}
|
|
599
|
+
if (!lane.needsResume) {
|
|
600
|
+
logger.success(`Lane '${options.lane}' is already completed.`);
|
|
601
|
+
return;
|
|
602
|
+
}
|
|
603
|
+
lanesToResume = [lane];
|
|
463
604
|
}
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
605
|
+
// Check for zombie lanes
|
|
606
|
+
const zombieCheck = checkAndFixZombieLanes(runDir);
|
|
607
|
+
if (zombieCheck.fixed.length > 0) {
|
|
608
|
+
logger.section('🔧 Zombie Lane Recovery');
|
|
609
|
+
logger.info(`Fixed ${zombieCheck.fixed.length} zombie lane(s): ${zombieCheck.fixed.join(', ')}`);
|
|
610
|
+
console.log('');
|
|
467
611
|
}
|
|
468
|
-
if (
|
|
469
|
-
|
|
612
|
+
if (options.status) {
|
|
613
|
+
printAllLaneStatus(runDir);
|
|
614
|
+
return;
|
|
470
615
|
}
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
logger.info('Running pre-flight checks...');
|
|
475
|
-
const report = (0, doctor_1.runDoctor)({
|
|
476
|
-
cwd: process.cwd(),
|
|
477
|
-
tasksDir,
|
|
478
|
-
includeCursorAgentChecks: false, // Skip agent checks for resume
|
|
479
|
-
});
|
|
480
|
-
// Only show blocking errors for resume
|
|
481
|
-
const blockingIssues = report.issues.filter(i => i.severity === 'error' &&
|
|
482
|
-
(i.id.startsWith('branch.') || i.id.startsWith('git.')));
|
|
483
|
-
if (blockingIssues.length > 0) {
|
|
484
|
-
logger.section('🛑 Pre-resume check found issues');
|
|
485
|
-
for (const issue of blockingIssues) {
|
|
486
|
-
logger.error(`${issue.title} (${issue.id})`, '❌');
|
|
487
|
-
console.log(` ${issue.message}`);
|
|
488
|
-
if (issue.details)
|
|
489
|
-
console.log(` Details: ${issue.details}`);
|
|
490
|
-
if (issue.fixes?.length) {
|
|
491
|
-
console.log(' Fix:');
|
|
492
|
-
for (const fix of issue.fixes)
|
|
493
|
-
console.log(` - ${fix}`);
|
|
494
|
-
}
|
|
495
|
-
console.log('');
|
|
496
|
-
}
|
|
497
|
-
throw new Error('Pre-resume checks failed. Use --skip-doctor to bypass (not recommended).');
|
|
616
|
+
if (lanesToResume.length === 0) {
|
|
617
|
+
if (options.lane || options.all) {
|
|
618
|
+
logger.success('No lanes need to be resumed.');
|
|
498
619
|
}
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
if (warnings.length > 0) {
|
|
502
|
-
logger.warn(`${warnings.length} warning(s) found. Run 'cursorflow doctor' for details.`);
|
|
620
|
+
else {
|
|
621
|
+
printAllLaneStatus(runDir);
|
|
503
622
|
}
|
|
623
|
+
return;
|
|
504
624
|
}
|
|
505
|
-
|
|
506
|
-
logger.info(`Run: ${path.basename(runDir)}`);
|
|
507
|
-
logger.info(`Tasks: ${state.tasksFile}`);
|
|
508
|
-
logger.info(`Starting from task index: ${options.restart ? 0 : state.currentTaskIndex}`);
|
|
509
|
-
const child = spawnLaneResume(options.lane, laneDir, state, {
|
|
625
|
+
const result = await resumeLanes(lanesToResume, allLanes, {
|
|
510
626
|
restart: options.restart,
|
|
627
|
+
maxConcurrent: options.maxConcurrent,
|
|
628
|
+
skipDoctor: options.skipDoctor,
|
|
629
|
+
noGit: options.noGit,
|
|
630
|
+
executor: options.executor,
|
|
631
|
+
enhancedLogConfig: config.enhancedLogging,
|
|
511
632
|
});
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
if (code === 0) {
|
|
516
|
-
logger.success(`Lane ${options.lane} completed successfully`);
|
|
517
|
-
resolve();
|
|
518
|
-
}
|
|
519
|
-
else if (code === 2) {
|
|
520
|
-
logger.warn(`Lane ${options.lane} blocked on dependency change`);
|
|
521
|
-
resolve();
|
|
522
|
-
}
|
|
523
|
-
else {
|
|
524
|
-
reject(new Error(`Lane ${options.lane} failed with exit code ${code}`));
|
|
525
|
-
}
|
|
526
|
-
});
|
|
527
|
-
child.on('error', (error) => {
|
|
528
|
-
reject(new Error(`Failed to start runner: ${error.message}`));
|
|
529
|
-
});
|
|
530
|
-
});
|
|
633
|
+
if (result.failed.length > 0) {
|
|
634
|
+
throw new Error(`${result.failed.length} lane(s) failed to complete`);
|
|
635
|
+
}
|
|
531
636
|
}
|
|
532
637
|
module.exports = resume;
|
|
533
638
|
//# sourceMappingURL=resume.js.map
|