@yemi33/minions 0.1.60 → 0.1.61
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +11 -0
- package/engine/cleanup.js +393 -0
- package/engine/dispatch.js +207 -0
- package/engine/timeout.js +280 -0
- package/engine.js +27 -757
- package/package.json +1 -1
package/CHANGELOG.md
CHANGED
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* engine/cleanup.js — Periodic cleanup: temp files, worktrees, zombies, migrations.
|
|
3
|
+
* Extracted from engine.js for modularity. No logic changes.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const shared = require('./shared');
|
|
9
|
+
const queries = require('./queries');
|
|
10
|
+
|
|
11
|
+
const { exec, execSilent } = shared;
|
|
12
|
+
const { safeJson, safeWrite, safeReadDir, getProjects, projectWorkItemsPath, projectPrPath,
|
|
13
|
+
sanitizeBranch, KB_CATEGORIES } = shared;
|
|
14
|
+
const { getDispatch, getAgentStatus } = queries;
|
|
15
|
+
|
|
16
|
+
const MINIONS_DIR = shared.MINIONS_DIR;
|
|
17
|
+
const AGENTS_DIR = queries.AGENTS_DIR;
|
|
18
|
+
const ENGINE_DIR = queries.ENGINE_DIR;
|
|
19
|
+
const PRD_DIR = queries.PRD_DIR;
|
|
20
|
+
const PLANS_DIR = queries.PLANS_DIR;
|
|
21
|
+
|
|
22
|
+
// Lazy require to break circular dependency with engine.js
|
|
23
|
+
let _engine = null;
|
|
24
|
+
function engine() { if (!_engine) _engine = require('../engine'); return _engine; }
|
|
25
|
+
function log(level, msg, meta) { return engine().log(level, msg, meta); }
|
|
26
|
+
function ts() { return engine().ts(); }
|
|
27
|
+
|
|
28
|
+
// Lazy require for dispatch module
|
|
29
|
+
let _dispatch = null;
|
|
30
|
+
function dispatchModule() { if (!_dispatch) _dispatch = require('./dispatch'); return _dispatch; }
|
|
31
|
+
|
|
32
|
+
// ─── Cleanup Orchestrator ────────────────────────────────────────────────────
|
|
33
|
+
|
|
34
|
+
function runCleanup(config, verbose = false) {
|
|
35
|
+
const activeProcesses = engine().activeProcesses;
|
|
36
|
+
const projects = getProjects(config);
|
|
37
|
+
let cleaned = { tempFiles: 0, liveOutputs: 0, worktrees: 0, zombies: 0 };
|
|
38
|
+
|
|
39
|
+
// 1. Clean stale temp prompt/sysprompt files (older than 1 hour)
|
|
40
|
+
const oneHourAgo = Date.now() - 3600000;
|
|
41
|
+
try {
|
|
42
|
+
const tmpDir = path.join(ENGINE_DIR, 'tmp');
|
|
43
|
+
const scanDirs = [ENGINE_DIR, ...(fs.existsSync(tmpDir) ? [tmpDir] : [])];
|
|
44
|
+
for (const dir of scanDirs) {
|
|
45
|
+
for (const f of fs.readdirSync(dir)) {
|
|
46
|
+
if (f.startsWith('prompt-') || f.startsWith('sysprompt-') || f.startsWith('tmp-sysprompt-')) {
|
|
47
|
+
const fp = path.join(dir, f);
|
|
48
|
+
try {
|
|
49
|
+
const stat = fs.statSync(fp);
|
|
50
|
+
if (stat.mtimeMs < oneHourAgo) {
|
|
51
|
+
fs.unlinkSync(fp);
|
|
52
|
+
cleaned.tempFiles++;
|
|
53
|
+
}
|
|
54
|
+
} catch { /* cleanup */ }
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
} catch (e) { log('warn', 'cleanup temp files: ' + e.message); }
|
|
59
|
+
|
|
60
|
+
// 2. Clean live-output.log for idle agents (not currently working)
|
|
61
|
+
for (const [agentId] of Object.entries(config.agents || {})) {
|
|
62
|
+
const status = getAgentStatus(agentId);
|
|
63
|
+
if (status.status !== 'working') {
|
|
64
|
+
const livePath = path.join(AGENTS_DIR, agentId, 'live-output.log');
|
|
65
|
+
if (fs.existsSync(livePath)) {
|
|
66
|
+
try {
|
|
67
|
+
const stat = fs.statSync(livePath);
|
|
68
|
+
if (stat.mtimeMs < oneHourAgo) {
|
|
69
|
+
fs.unlinkSync(livePath);
|
|
70
|
+
cleaned.liveOutputs++;
|
|
71
|
+
}
|
|
72
|
+
} catch { /* cleanup */ }
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
// 3. Clean git worktrees for merged/abandoned PRs
|
|
78
|
+
for (const project of projects) {
|
|
79
|
+
const root = project.localPath ? path.resolve(project.localPath) : null;
|
|
80
|
+
if (!root || !fs.existsSync(root)) continue;
|
|
81
|
+
|
|
82
|
+
const worktreeRoot = path.resolve(root, config.engine?.worktreeRoot || '../worktrees');
|
|
83
|
+
if (!fs.existsSync(worktreeRoot)) continue;
|
|
84
|
+
|
|
85
|
+
// Get PRs for this project
|
|
86
|
+
const prs = safeJson(projectPrPath(project)) || [];
|
|
87
|
+
const mergedBranches = new Set();
|
|
88
|
+
for (const pr of prs) {
|
|
89
|
+
if (pr.status === 'merged' || pr.status === 'abandoned' || pr.status === 'completed') {
|
|
90
|
+
if (pr.branch) mergedBranches.add(pr.branch);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// List worktrees — collect info for age-based + cap-based cleanup
|
|
95
|
+
const MAX_WORKTREES = 10;
|
|
96
|
+
try {
|
|
97
|
+
const dirs = fs.readdirSync(worktreeRoot);
|
|
98
|
+
const wtEntries = []; // { dir, wtPath, mtime, shouldClean, isProtected }
|
|
99
|
+
const dispatch = getDispatch();
|
|
100
|
+
|
|
101
|
+
for (const dir of dirs) {
|
|
102
|
+
const wtPath = path.join(worktreeRoot, dir);
|
|
103
|
+
try { if (!fs.statSync(wtPath).isDirectory()) continue; } catch { continue; }
|
|
104
|
+
|
|
105
|
+
let shouldClean = false;
|
|
106
|
+
let isProtected = false;
|
|
107
|
+
|
|
108
|
+
// Check if this worktree's branch is merged/abandoned
|
|
109
|
+
// Use sanitized exact match on the branch portion of the dir name (format: {slug}-{branch}-{suffix})
|
|
110
|
+
const dirLower = dir.toLowerCase();
|
|
111
|
+
for (const branch of mergedBranches) {
|
|
112
|
+
const branchSlug = sanitizeBranch(branch).toLowerCase();
|
|
113
|
+
if (dirLower === branchSlug || dirLower.includes(branchSlug + '-') || dirLower.endsWith('-' + branchSlug)) {
|
|
114
|
+
shouldClean = true;
|
|
115
|
+
break;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
// Check if referenced by active/pending dispatch (use sanitized branch comparison)
|
|
120
|
+
const isReferenced = [...dispatch.pending, ...(dispatch.active || [])].some(d => {
|
|
121
|
+
if (!d.meta?.branch) return false;
|
|
122
|
+
const dispBranch = sanitizeBranch(d.meta.branch).toLowerCase();
|
|
123
|
+
return dirLower.includes(dispBranch);
|
|
124
|
+
});
|
|
125
|
+
if (isReferenced) isProtected = true;
|
|
126
|
+
|
|
127
|
+
// Also clean worktrees older than 2 hours with no active dispatch referencing them
|
|
128
|
+
let mtime = Date.now();
|
|
129
|
+
if (!shouldClean) {
|
|
130
|
+
try {
|
|
131
|
+
const stat = fs.statSync(wtPath);
|
|
132
|
+
mtime = stat.mtimeMs;
|
|
133
|
+
const ageMs = Date.now() - mtime;
|
|
134
|
+
if (ageMs > 7200000 && !isReferenced) { // 2 hours
|
|
135
|
+
shouldClean = true;
|
|
136
|
+
}
|
|
137
|
+
} catch { /* optional */ }
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// Skip worktrees for active shared-branch plans (check both prd/ and plans/ for .json PRDs)
|
|
141
|
+
if (shouldClean || !isProtected) {
|
|
142
|
+
try {
|
|
143
|
+
for (const checkDir of [PRD_DIR, path.join(MINIONS_DIR, 'plans')]) {
|
|
144
|
+
if (!fs.existsSync(checkDir)) continue;
|
|
145
|
+
for (const pf of fs.readdirSync(checkDir).filter(f => f.endsWith('.json'))) {
|
|
146
|
+
const plan = safeJson(path.join(checkDir, pf));
|
|
147
|
+
if (plan?.branch_strategy === 'shared-branch' && plan?.feature_branch && plan?.status !== 'completed') {
|
|
148
|
+
const planBranch = sanitizeBranch(plan.feature_branch).toLowerCase();
|
|
149
|
+
if (dirLower.includes(planBranch)) {
|
|
150
|
+
isProtected = true;
|
|
151
|
+
if (shouldClean) {
|
|
152
|
+
shouldClean = false;
|
|
153
|
+
if (verbose) console.log(` Skipping worktree ${dir}: active shared-branch plan`);
|
|
154
|
+
}
|
|
155
|
+
break;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
if (isProtected) break;
|
|
160
|
+
}
|
|
161
|
+
} catch (e) { log('warn', 'check shared-branch protection: ' + e.message); }
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
wtEntries.push({ dir, wtPath, mtime, shouldClean, isProtected });
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
// Enforce max worktree cap — if over limit, mark oldest unprotected for cleanup
|
|
168
|
+
const surviving = wtEntries.filter(e => !e.shouldClean && !e.isProtected);
|
|
169
|
+
if (surviving.length + wtEntries.filter(e => e.isProtected).length > MAX_WORKTREES) {
|
|
170
|
+
// Sort oldest first
|
|
171
|
+
surviving.sort((a, b) => a.mtime - b.mtime);
|
|
172
|
+
const excess = surviving.length + wtEntries.filter(e => e.isProtected).length - MAX_WORKTREES;
|
|
173
|
+
for (let i = 0; i < Math.min(excess, surviving.length); i++) {
|
|
174
|
+
surviving[i].shouldClean = true;
|
|
175
|
+
if (verbose) console.log(` Marking worktree ${surviving[i].dir} for cap cleanup (${MAX_WORKTREES} max)`);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
// Remove all marked worktrees
|
|
180
|
+
for (const entry of wtEntries) {
|
|
181
|
+
if (entry.shouldClean) {
|
|
182
|
+
try {
|
|
183
|
+
exec(`git worktree remove "${entry.wtPath}" --force`, { cwd: root, stdio: 'pipe' });
|
|
184
|
+
cleaned.worktrees++;
|
|
185
|
+
if (verbose) console.log(` Removed worktree: ${entry.wtPath}`);
|
|
186
|
+
} catch (e) {
|
|
187
|
+
if (verbose) console.log(` Failed to remove worktree ${entry.wtPath}: ${e.message}`);
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
} catch (e) { log('warn', 'cleanup worktrees: ' + e.message); }
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
// 4. Kill zombie claude processes not tracked by the engine
|
|
195
|
+
// List all node processes, check if any are running spawn-agent.js for our minions
|
|
196
|
+
try {
|
|
197
|
+
const dispatch = getDispatch();
|
|
198
|
+
const activePids = new Set();
|
|
199
|
+
for (const [, info] of activeProcesses.entries()) {
|
|
200
|
+
if (info.proc?.pid) activePids.add(info.proc.pid);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
// Clean individual orphaned processes — no matching active dispatch
|
|
204
|
+
const activeIds = new Set((dispatch.active || []).map(d => d.id));
|
|
205
|
+
for (const [id, info] of activeProcesses.entries()) {
|
|
206
|
+
if (!activeIds.has(id)) {
|
|
207
|
+
try { if (info.proc) info.proc.kill('SIGTERM'); } catch { /* process may be dead */ }
|
|
208
|
+
activeProcesses.delete(id);
|
|
209
|
+
cleaned.zombies++;
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
} catch (e) { log('warn', 'cleanup zombie processes: ' + e.message); }
|
|
213
|
+
|
|
214
|
+
// 5. Clean spawn-debug.log
|
|
215
|
+
try { fs.unlinkSync(path.join(ENGINE_DIR, 'spawn-debug.log')); } catch { /* cleanup */ }
|
|
216
|
+
|
|
217
|
+
// 6. Prune old output archive files (keep last 30 per agent)
|
|
218
|
+
for (const agentId of Object.keys(config.agents || {})) {
|
|
219
|
+
const agentDir = path.join(MINIONS_DIR, 'agents', agentId);
|
|
220
|
+
if (!fs.existsSync(agentDir)) continue;
|
|
221
|
+
try {
|
|
222
|
+
const outputFiles = fs.readdirSync(agentDir)
|
|
223
|
+
.filter(f => f.startsWith('output-') && f.endsWith('.log') && f !== 'output.log')
|
|
224
|
+
.map(f => ({ name: f, mtime: fs.statSync(path.join(agentDir, f)).mtimeMs }))
|
|
225
|
+
.sort((a, b) => b.mtime - a.mtime);
|
|
226
|
+
for (const old of outputFiles.slice(30)) {
|
|
227
|
+
try { fs.unlinkSync(path.join(agentDir, old.name)); cleaned.files++; } catch { /* cleanup */ }
|
|
228
|
+
}
|
|
229
|
+
} catch (e) { log('warn', 'prune output archives: ' + e.message); }
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
// 7. Prune orphaned dispatch entries — items whose source work item no longer exists
|
|
233
|
+
cleaned.orphanedDispatches = 0;
|
|
234
|
+
try {
|
|
235
|
+
const dispatch = getDispatch();
|
|
236
|
+
// Collect all work item IDs across all sources
|
|
237
|
+
const allWiIds = new Set();
|
|
238
|
+
try {
|
|
239
|
+
const central = safeJson(path.join(MINIONS_DIR, 'work-items.json')) || [];
|
|
240
|
+
central.forEach(w => allWiIds.add(w.id));
|
|
241
|
+
} catch (e) { log('warn', 'read central work items for orphan check: ' + e.message); }
|
|
242
|
+
for (const project of projects) {
|
|
243
|
+
try {
|
|
244
|
+
const projItems = safeJson(projectWorkItemsPath(project)) || [];
|
|
245
|
+
projItems.forEach(w => allWiIds.add(w.id));
|
|
246
|
+
} catch (e) { log('warn', 'read project work items for orphan check: ' + e.message); }
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
let changed = false;
|
|
250
|
+
for (const queue of ['pending', 'active']) {
|
|
251
|
+
if (!dispatch[queue]) continue;
|
|
252
|
+
const before = dispatch[queue].length;
|
|
253
|
+
dispatch[queue] = dispatch[queue].filter(d => {
|
|
254
|
+
const itemId = d.meta?.item?.id;
|
|
255
|
+
if (!itemId) return true; // keep entries without item tracking
|
|
256
|
+
return allWiIds.has(itemId);
|
|
257
|
+
});
|
|
258
|
+
const removed = before - dispatch[queue].length;
|
|
259
|
+
if (removed > 0) {
|
|
260
|
+
cleaned.orphanedDispatches += removed;
|
|
261
|
+
changed = true;
|
|
262
|
+
}
|
|
263
|
+
}
|
|
264
|
+
if (changed) {
|
|
265
|
+
const { mutateDispatch } = dispatchModule();
|
|
266
|
+
mutateDispatch((dp) => {
|
|
267
|
+
for (const queue of ['pending', 'active']) {
|
|
268
|
+
if (!dp[queue]) continue;
|
|
269
|
+
dp[queue] = dp[queue].filter(d => {
|
|
270
|
+
const itemId = d.meta?.item?.id;
|
|
271
|
+
if (!itemId) return true;
|
|
272
|
+
return allWiIds.has(itemId);
|
|
273
|
+
});
|
|
274
|
+
}
|
|
275
|
+
});
|
|
276
|
+
}
|
|
277
|
+
} catch (e) { log('warn', 'prune orphaned dispatches: ' + e.message); }
|
|
278
|
+
|
|
279
|
+
if (cleaned.tempFiles + cleaned.liveOutputs + cleaned.worktrees + cleaned.zombies + (cleaned.files || 0) + cleaned.orphanedDispatches > 0) {
|
|
280
|
+
log('info', `Cleanup: ${cleaned.tempFiles} temp, ${cleaned.liveOutputs} live outputs, ${cleaned.worktrees} worktrees, ${cleaned.zombies} zombies, ${cleaned.files || 0} archives, ${cleaned.orphanedDispatches} orphaned dispatches`);
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
// 8. Clean swept KB files older than 7 days
|
|
284
|
+
try {
|
|
285
|
+
const sweptDir = path.join(MINIONS_DIR, 'knowledge', '_swept');
|
|
286
|
+
if (fs.existsSync(sweptDir)) {
|
|
287
|
+
const sevenDaysAgo = Date.now() - 7 * 86400000;
|
|
288
|
+
for (const f of fs.readdirSync(sweptDir)) {
|
|
289
|
+
try {
|
|
290
|
+
const fp = path.join(sweptDir, f);
|
|
291
|
+
if (fs.statSync(fp).mtimeMs < sevenDaysAgo) {
|
|
292
|
+
fs.unlinkSync(fp);
|
|
293
|
+
if (!cleaned.sweptKb) cleaned.sweptKb = 0;
|
|
294
|
+
cleaned.sweptKb++;
|
|
295
|
+
}
|
|
296
|
+
} catch { /* cleanup */ }
|
|
297
|
+
}
|
|
298
|
+
}
|
|
299
|
+
} catch (e) { log('warn', 'cleanup swept KB files: ' + e.message); }
|
|
300
|
+
|
|
301
|
+
// 9. KB watchdog — restore deleted KB files from git if count dropped vs checkpoint
|
|
302
|
+
try {
|
|
303
|
+
const checkpoint = safeJson(path.join(ENGINE_DIR, 'kb-checkpoint.json'));
|
|
304
|
+
if (checkpoint && checkpoint.count > 0) {
|
|
305
|
+
const cats = KB_CATEGORIES;
|
|
306
|
+
const knowledgeDir = path.join(MINIONS_DIR, 'knowledge');
|
|
307
|
+
let current = 0;
|
|
308
|
+
for (const cat of cats) {
|
|
309
|
+
const d = path.join(knowledgeDir, cat);
|
|
310
|
+
if (fs.existsSync(d)) current += fs.readdirSync(d).length;
|
|
311
|
+
}
|
|
312
|
+
if (current < checkpoint.count) {
|
|
313
|
+
log('warn', `KB watchdog: file count dropped ${checkpoint.count} → ${current}, restoring from git`);
|
|
314
|
+
try {
|
|
315
|
+
const trackedCheck = execSilent('git ls-tree --name-only HEAD -- knowledge', { cwd: MINIONS_DIR }).toString().trim();
|
|
316
|
+
if (!trackedCheck) {
|
|
317
|
+
log('warn', 'KB watchdog: knowledge/ is not tracked in git HEAD — skipping restore');
|
|
318
|
+
} else {
|
|
319
|
+
execSilent('git checkout HEAD -- knowledge', { cwd: MINIONS_DIR });
|
|
320
|
+
log('info', 'KB watchdog: restored knowledge/ from git HEAD');
|
|
321
|
+
}
|
|
322
|
+
} catch (err) {
|
|
323
|
+
log('error', `KB watchdog: git restore failed — ${err.message}`);
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
}
|
|
327
|
+
} catch (e) { log('warn', 'KB watchdog check: ' + e.message); }
|
|
328
|
+
|
|
329
|
+
// 6. Migrate legacy work-item statuses to canonical values
|
|
330
|
+
// in-pr, implemented, complete → done (one-time correction per item)
|
|
331
|
+
const LEGACY_DONE_STATUSES = new Set(['in-pr', 'implemented', 'complete']);
|
|
332
|
+
for (const project of projects) {
|
|
333
|
+
try {
|
|
334
|
+
const wiPath = projectWorkItemsPath(project);
|
|
335
|
+
const items = safeJson(wiPath) || [];
|
|
336
|
+
let migrated = 0;
|
|
337
|
+
for (const item of items) {
|
|
338
|
+
if (LEGACY_DONE_STATUSES.has(item.status)) {
|
|
339
|
+
item.status = 'done';
|
|
340
|
+
migrated++;
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
if (migrated > 0) {
|
|
344
|
+
safeWrite(wiPath, items);
|
|
345
|
+
log('info', `Migrated ${migrated} legacy status(es) → done in ${project.name} work items`);
|
|
346
|
+
}
|
|
347
|
+
} catch (e) { log('warn', 'migrate legacy statuses: ' + e.message); }
|
|
348
|
+
}
|
|
349
|
+
// Central work items
|
|
350
|
+
try {
|
|
351
|
+
const centralPath = path.join(MINIONS_DIR, 'work-items.json');
|
|
352
|
+
const centralItems = safeJson(centralPath) || [];
|
|
353
|
+
let migrated = 0;
|
|
354
|
+
for (const item of centralItems) {
|
|
355
|
+
if (LEGACY_DONE_STATUSES.has(item.status)) {
|
|
356
|
+
item.status = 'done';
|
|
357
|
+
migrated++;
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
if (migrated > 0) {
|
|
361
|
+
safeWrite(centralPath, centralItems);
|
|
362
|
+
log('info', `Migrated ${migrated} legacy status(es) → done in central work items`);
|
|
363
|
+
}
|
|
364
|
+
} catch (e) { log('warn', 'migrate central legacy statuses: ' + e.message); }
|
|
365
|
+
// PRD items (missing_features[].status)
|
|
366
|
+
try {
|
|
367
|
+
const prdFiles = fs.readdirSync(PRD_DIR).filter(f => f.endsWith('.json'));
|
|
368
|
+
for (const pf of prdFiles) {
|
|
369
|
+
const prdPath = path.join(PRD_DIR, pf);
|
|
370
|
+
const prd = safeJson(prdPath);
|
|
371
|
+
if (!prd?.missing_features) continue;
|
|
372
|
+
let migrated = 0;
|
|
373
|
+
for (const feat of prd.missing_features) {
|
|
374
|
+
if (LEGACY_DONE_STATUSES.has(feat.status)) {
|
|
375
|
+
feat.status = 'done';
|
|
376
|
+
migrated++;
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
if (migrated > 0) {
|
|
380
|
+
safeWrite(prdPath, prd);
|
|
381
|
+
log('info', `Migrated ${migrated} legacy PRD item status(es) → done in ${pf}`);
|
|
382
|
+
}
|
|
383
|
+
}
|
|
384
|
+
} catch (e) { log('warn', 'migrate PRD legacy statuses: ' + e.message); }
|
|
385
|
+
|
|
386
|
+
return cleaned;
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
// ─── Exports ─────────────────────────────────────────────────────────────────
|
|
390
|
+
|
|
391
|
+
module.exports = {
|
|
392
|
+
runCleanup,
|
|
393
|
+
};
|
|
@@ -0,0 +1,207 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* engine/dispatch.js — Dispatch queue management: add, complete, mutate, alerts.
|
|
3
|
+
* Extracted from engine.js for modularity. No logic changes.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
const fs = require('fs');
|
|
7
|
+
const path = require('path');
|
|
8
|
+
const shared = require('./shared');
|
|
9
|
+
const queries = require('./queries');
|
|
10
|
+
const { setCooldownFailure } = require('./cooldown');
|
|
11
|
+
|
|
12
|
+
const { safeJson, safeWrite, safeReadDir, mutateJsonFileLocked,
|
|
13
|
+
getProjects, projectWorkItemsPath } = shared;
|
|
14
|
+
const { getConfig, getDispatch, DISPATCH_PATH, INBOX_DIR } = queries;
|
|
15
|
+
|
|
16
|
+
const MINIONS_DIR = shared.MINIONS_DIR;
|
|
17
|
+
|
|
18
|
+
// Lazy require to break circular dependency with engine.js
|
|
19
|
+
let _lifecycle = null;
|
|
20
|
+
function lifecycle() { if (!_lifecycle) _lifecycle = require('./lifecycle'); return _lifecycle; }
|
|
21
|
+
|
|
22
|
+
// ─── Engine utilities (lazy require to avoid circular deps) ──────────────────
|
|
23
|
+
let _engine = null;
|
|
24
|
+
function engine() { if (!_engine) _engine = require('../engine'); return _engine; }
|
|
25
|
+
function log(level, msg, meta) { return engine().log(level, msg, meta); }
|
|
26
|
+
function ts() { return engine().ts(); }
|
|
27
|
+
function dateStamp() { return new Date().toISOString().slice(0, 10); }
|
|
28
|
+
|
|
29
|
+
// ─── Dispatch Mutation ───────────────────────────────────────────────────────
|
|
30
|
+
|
|
31
|
+
function mutateDispatch(mutator) {
|
|
32
|
+
const defaultDispatch = { pending: [], active: [], completed: [] };
|
|
33
|
+
return mutateJsonFileLocked(DISPATCH_PATH, (dispatch) => {
|
|
34
|
+
dispatch.pending = Array.isArray(dispatch.pending) ? dispatch.pending : [];
|
|
35
|
+
dispatch.active = Array.isArray(dispatch.active) ? dispatch.active : [];
|
|
36
|
+
dispatch.completed = Array.isArray(dispatch.completed) ? dispatch.completed : [];
|
|
37
|
+
return mutator(dispatch) || dispatch;
|
|
38
|
+
}, { defaultValue: defaultDispatch });
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
// ─── Add to Dispatch ─────────────────────────────────────────────────────────
|
|
42
|
+
|
|
43
|
+
function addToDispatch(item) {
|
|
44
|
+
item.id = item.id || `${item.agent}-${item.type}-${shared.uid()}`;
|
|
45
|
+
item.created_at = ts();
|
|
46
|
+
mutateDispatch((dispatch) => {
|
|
47
|
+
dispatch.pending.push(item);
|
|
48
|
+
});
|
|
49
|
+
log('info', `Queued dispatch: ${item.id} (${item.type} → ${item.agent})`);
|
|
50
|
+
return item.id;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// ─── Retryable Failure Classification ────────────────────────────────────────
|
|
54
|
+
|
|
55
|
+
function isRetryableFailureReason(reason = '') {
|
|
56
|
+
const r = String(reason || '').toLowerCase();
|
|
57
|
+
if (!r) return true; // unknown error from tool exit — keep retryable
|
|
58
|
+
const nonRetryable = [
|
|
59
|
+
'no playbook rendered',
|
|
60
|
+
'failed to render',
|
|
61
|
+
'no target project available',
|
|
62
|
+
'no plan files found',
|
|
63
|
+
'plan file not found',
|
|
64
|
+
'invalid filename',
|
|
65
|
+
'invalid file path',
|
|
66
|
+
'missing required',
|
|
67
|
+
'validation failed',
|
|
68
|
+
];
|
|
69
|
+
return !nonRetryable.some(s => r.includes(s));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
// ─── Complete Dispatch ───────────────────────────────────────────────────────
|
|
73
|
+
|
|
74
|
+
function completeDispatch(id, result = 'success', reason = '', resultSummary = '', opts = {}) {
|
|
75
|
+
const { processWorkItemFailure = true } = opts;
|
|
76
|
+
let item = null;
|
|
77
|
+
|
|
78
|
+
mutateDispatch((dispatch) => {
|
|
79
|
+
// Check active list first
|
|
80
|
+
let idx = dispatch.active.findIndex(d => d.id === id);
|
|
81
|
+
if (idx >= 0) {
|
|
82
|
+
item = dispatch.active.splice(idx, 1)[0];
|
|
83
|
+
} else {
|
|
84
|
+
// Also check pending list (e.g., worktree failure before spawn)
|
|
85
|
+
idx = dispatch.pending.findIndex(d => d.id === id);
|
|
86
|
+
if (idx >= 0) item = dispatch.pending.splice(idx, 1)[0];
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
if (!item) return;
|
|
90
|
+
item.completed_at = ts();
|
|
91
|
+
item.result = result;
|
|
92
|
+
if (reason) item.reason = reason;
|
|
93
|
+
if (resultSummary) item.resultSummary = resultSummary;
|
|
94
|
+
delete item.prompt;
|
|
95
|
+
if (dispatch.completed.length >= 100) {
|
|
96
|
+
dispatch.completed = dispatch.completed.slice(-99);
|
|
97
|
+
}
|
|
98
|
+
dispatch.completed.push(item);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
if (item) {
|
|
102
|
+
log('info', `Completed dispatch: ${id} (${result}${reason ? ': ' + reason : ''})`);
|
|
103
|
+
|
|
104
|
+
// Update source work item status on failure + auto-retry with backoff
|
|
105
|
+
const retryableFailure = isRetryableFailureReason(reason);
|
|
106
|
+
if (result === 'error' && item.meta?.dispatchKey && retryableFailure) setCooldownFailure(item.meta.dispatchKey);
|
|
107
|
+
|
|
108
|
+
if (processWorkItemFailure && result === 'error' && item.meta?.item?.id) {
|
|
109
|
+
let retries = (item.meta.item._retryCount || 0);
|
|
110
|
+
try {
|
|
111
|
+
const wiPath = item.meta.source === 'central-work-item' || item.meta.source === 'central-work-item-fanout'
|
|
112
|
+
? path.join(MINIONS_DIR, 'work-items.json')
|
|
113
|
+
: item.meta.project?.name ? projectWorkItemsPath({ name: item.meta.project.name, localPath: item.meta.project.localPath }) : null;
|
|
114
|
+
if (wiPath) {
|
|
115
|
+
const items = safeJson(wiPath) || [];
|
|
116
|
+
const wi = items.find(i => i.id === item.meta.item.id);
|
|
117
|
+
if (wi) retries = wi._retryCount || 0;
|
|
118
|
+
}
|
|
119
|
+
} catch (e) { log('warn', 'read retry count: ' + e.message); }
|
|
120
|
+
if (retryableFailure && retries < 3) {
|
|
121
|
+
log('info', `Dispatch error for ${item.meta.item.id} — auto-retry ${retries + 1}/3`);
|
|
122
|
+
lifecycle().updateWorkItemStatus(item.meta, 'pending', '');
|
|
123
|
+
// Remove this dispatch key from completed so dedupe doesn't block immediate redispatch.
|
|
124
|
+
if (item.meta?.dispatchKey) {
|
|
125
|
+
try {
|
|
126
|
+
mutateDispatch((dp) => {
|
|
127
|
+
dp.completed = Array.isArray(dp.completed) ? dp.completed.filter(d => d.meta?.dispatchKey !== item.meta.dispatchKey) : [];
|
|
128
|
+
return dp;
|
|
129
|
+
});
|
|
130
|
+
} catch (e) { log('warn', 'clear dispatch for retry: ' + e.message); }
|
|
131
|
+
}
|
|
132
|
+
// Increment retry counter on the source work item
|
|
133
|
+
try {
|
|
134
|
+
const wiPath = item.meta.source === 'central-work-item' || item.meta.source === 'central-work-item-fanout'
|
|
135
|
+
? path.join(MINIONS_DIR, 'work-items.json')
|
|
136
|
+
: item.meta.project?.name ? projectWorkItemsPath({ name: item.meta.project.name, localPath: item.meta.project.localPath }) : null;
|
|
137
|
+
if (wiPath) {
|
|
138
|
+
const items = safeJson(wiPath) || [];
|
|
139
|
+
const wi = items.find(i => i.id === item.meta.item.id);
|
|
140
|
+
if (wi && wi.status !== 'paused') {
|
|
141
|
+
wi._retryCount = retries + 1;
|
|
142
|
+
wi.status = 'pending';
|
|
143
|
+
wi._lastRetryReason = reason || '';
|
|
144
|
+
wi._lastRetryAt = ts();
|
|
145
|
+
delete wi.failReason;
|
|
146
|
+
delete wi.failedAt;
|
|
147
|
+
delete wi.dispatched_at;
|
|
148
|
+
delete wi.dispatched_to;
|
|
149
|
+
safeWrite(wiPath, items);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
} catch (e) { log('warn', 'increment retry counter: ' + e.message); }
|
|
153
|
+
} else {
|
|
154
|
+
const finalReason = !retryableFailure
|
|
155
|
+
? `Non-retryable failure: ${reason || 'Unknown error'}`
|
|
156
|
+
: (reason || 'Failed after 3 retries');
|
|
157
|
+
lifecycle().updateWorkItemStatus(item.meta, 'failed', finalReason);
|
|
158
|
+
// Alert: find items blocked by this failure and write inbox note
|
|
159
|
+
try {
|
|
160
|
+
const config = getConfig();
|
|
161
|
+
const failedId = item.meta.item.id;
|
|
162
|
+
const blockedItems = [];
|
|
163
|
+
for (const p of getProjects(config)) {
|
|
164
|
+
const items = safeJson(projectWorkItemsPath(p)) || [];
|
|
165
|
+
items.filter(w => w.status === 'pending' && (w.depends_on || []).includes(failedId))
|
|
166
|
+
.forEach(w => blockedItems.push(`- \`${w.id}\` — ${w.title}`));
|
|
167
|
+
}
|
|
168
|
+
const centralItems = safeJson(path.join(MINIONS_DIR, 'work-items.json')) || [];
|
|
169
|
+
centralItems.filter(w => w.status === 'pending' && (w.depends_on || []).includes(failedId))
|
|
170
|
+
.forEach(w => blockedItems.push(`- \`${w.id}\` — ${w.title}`));
|
|
171
|
+
|
|
172
|
+
writeInboxAlert(`failed-${failedId}`,
|
|
173
|
+
`# Work Item Failed — \`${failedId}\`\n\n` +
|
|
174
|
+
`**Item:** ${item.meta.item.title || failedId}\n` +
|
|
175
|
+
`**Reason:** ${finalReason}\n\n` +
|
|
176
|
+
(blockedItems.length > 0
|
|
177
|
+
? `**Blocked dependents (${blockedItems.length}):**\n${blockedItems.join('\n')}\n\n` +
|
|
178
|
+
`These items cannot dispatch until \`${failedId}\` is fixed and reset to \`pending\`.\n`
|
|
179
|
+
: `No downstream items are blocked.\n`)
|
|
180
|
+
);
|
|
181
|
+
} catch (e) { log('warn', 'write failure alert: ' + e.message); }
|
|
182
|
+
}
|
|
183
|
+
}
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
// ─── Inbox Alert ─────────────────────────────────────────────────────────────
|
|
188
|
+
|
|
189
|
+
function writeInboxAlert(slug, content) {
|
|
190
|
+
try {
|
|
191
|
+
const file = path.join(INBOX_DIR, `engine-alert-${slug}-${dateStamp()}.md`);
|
|
192
|
+
// Dedupe: don't write the same alert twice in the same day
|
|
193
|
+
const existing = safeReadDir(INBOX_DIR).find(f => f.startsWith(`engine-alert-${slug}-${dateStamp()}`));
|
|
194
|
+
if (existing) return;
|
|
195
|
+
safeWrite(file, content);
|
|
196
|
+
} catch (e) { log('warn', 'write inbox alert: ' + e.message); }
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
// ─── Exports ─────────────────────────────────────────────────────────────────
|
|
200
|
+
|
|
201
|
+
module.exports = {
|
|
202
|
+
mutateDispatch,
|
|
203
|
+
addToDispatch,
|
|
204
|
+
isRetryableFailureReason,
|
|
205
|
+
completeDispatch,
|
|
206
|
+
writeInboxAlert,
|
|
207
|
+
};
|