@yemi33/minions 0.1.2381 → 0.1.2382
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/minions.js +41 -2
- package/dashboard/js/refresh.js +8 -2
- package/dashboard/js/render-other.js +38 -0
- package/dashboard/js/render-work-items.js +61 -20
- package/dashboard/js/settings.js +7 -8
- package/dashboard/pages/engine.html +6 -0
- package/dashboard.js +84 -15
- package/docs/README.md +1 -0
- package/docs/claude-md-propagation.md +118 -0
- package/docs/engine-restart.md +10 -0
- package/docs/worktree-lifecycle.md +9 -0
- package/engine/claude-md-context.js +195 -0
- package/engine/consolidation.js +47 -3
- package/engine/db/migrations/026-pre-dispatch-rejections.js +55 -0
- package/engine/dispatch.js +73 -21
- package/engine/lifecycle.js +70 -12
- package/engine/pipeline.js +147 -20
- package/engine/playbook.js +35 -0
- package/engine/prd-store.js +14 -6
- package/engine/quarantine-refs.js +33 -0
- package/engine/queries.js +8 -6
- package/engine/restart-health.js +69 -4
- package/engine/runtimes/claude.js +6 -0
- package/engine/runtimes/codex.js +18 -0
- package/engine/runtimes/copilot.js +7 -0
- package/engine/shared.js +63 -44
- package/engine/supervisor.js +72 -16
- package/engine/timeout.js +87 -16
- package/engine/untrusted-fence.js +2 -1
- package/engine.js +251 -16
- package/package.json +1 -1
package/engine/restart-health.js
CHANGED
|
@@ -32,6 +32,14 @@ function readEngineControl(minionsHome) {
|
|
|
32
32
|
}
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
function readSupervisorPid(minionsHome) {
|
|
36
|
+
try {
|
|
37
|
+
return normalizePid(fs.readFileSync(path.join(minionsHome, 'engine', 'supervisor.pid'), 'utf8').trim());
|
|
38
|
+
} catch {
|
|
39
|
+
return null;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
35
43
|
function normalizePid(pid) {
|
|
36
44
|
const n = Number(pid);
|
|
37
45
|
return Number.isInteger(n) && n > 0 ? n : null;
|
|
@@ -122,7 +130,10 @@ async function checkRestartHealth(options = {}) {
|
|
|
122
130
|
dashboardUrl,
|
|
123
131
|
dashboardPid,
|
|
124
132
|
dashboardPort = 7331,
|
|
133
|
+
expectedEnginePid,
|
|
134
|
+
supervisorPid,
|
|
125
135
|
readControl = readEngineControl,
|
|
136
|
+
readSupervisorPid: readSupPid = readSupervisorPid,
|
|
126
137
|
isProcessAlive: isAlive = isProcessAlive,
|
|
127
138
|
isPortListening: portCheck = isPortListening,
|
|
128
139
|
httpGetJson: getJson = httpGetJson,
|
|
@@ -130,8 +141,10 @@ async function checkRestartHealth(options = {}) {
|
|
|
130
141
|
|
|
131
142
|
const control = readControl(minionsHome);
|
|
132
143
|
const pid = normalizePid(control && control.pid);
|
|
144
|
+
const expectedEngine = normalizePid(expectedEnginePid);
|
|
133
145
|
const engineAlive = pid ? isAlive(pid) : false;
|
|
134
|
-
const
|
|
146
|
+
const engineOwned = expectedEngine == null || pid === expectedEngine;
|
|
147
|
+
const engineOk = control && control.state === 'running' && engineAlive && engineOwned;
|
|
135
148
|
|
|
136
149
|
// Two strategies for the dashboard check:
|
|
137
150
|
// 1) Process + port-listening (preferred from `minions restart` since
|
|
@@ -196,11 +209,54 @@ async function checkRestartHealth(options = {}) {
|
|
|
196
209
|
dashboardSnapshot = { kind: 'http', url, ok: !!(dashboard && dashboard.ok), statusCode: dashboard && dashboard.statusCode, status: dashboardStatus };
|
|
197
210
|
}
|
|
198
211
|
|
|
212
|
+
let supervisorOk = true;
|
|
213
|
+
let supervisorSnapshot = null;
|
|
214
|
+
if (supervisorPid != null) {
|
|
215
|
+
const expectedSupervisor = normalizePid(supervisorPid);
|
|
216
|
+
const recordedSupervisor = normalizePid(readSupPid(minionsHome));
|
|
217
|
+
const supervisorAlive = expectedSupervisor ? isAlive(expectedSupervisor) : false;
|
|
218
|
+
supervisorOk = !!(expectedSupervisor && supervisorAlive && recordedSupervisor === expectedSupervisor);
|
|
219
|
+
supervisorSnapshot = {
|
|
220
|
+
pid: expectedSupervisor,
|
|
221
|
+
alive: supervisorAlive,
|
|
222
|
+
recordedPid: recordedSupervisor,
|
|
223
|
+
owned: recordedSupervisor === expectedSupervisor,
|
|
224
|
+
};
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
let topologyOk = true;
|
|
228
|
+
let topologySnapshot = null;
|
|
229
|
+
if (engineOk && dashboardOk && supervisorOk && options.requireExactTopology) {
|
|
230
|
+
const listPids = options.listProcessPids;
|
|
231
|
+
const scripts = options.topologyScripts || {};
|
|
232
|
+
if (typeof listPids !== 'function') {
|
|
233
|
+
topologyOk = false;
|
|
234
|
+
topologySnapshot = { error: 'process enumerator unavailable' };
|
|
235
|
+
} else {
|
|
236
|
+
const expected = {
|
|
237
|
+
engine: expectedEngine,
|
|
238
|
+
dashboard: normalizePid(dashboardPid),
|
|
239
|
+
supervisor: normalizePid(supervisorPid),
|
|
240
|
+
};
|
|
241
|
+
const actual = {};
|
|
242
|
+
for (const kind of ['engine', 'dashboard', 'supervisor']) {
|
|
243
|
+
try {
|
|
244
|
+
actual[kind] = [...new Set((listPids(scripts[kind]) || []).map(normalizePid).filter(Boolean))];
|
|
245
|
+
} catch {
|
|
246
|
+
actual[kind] = [];
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
topologyOk = Object.keys(expected).every(kind =>
|
|
250
|
+
expected[kind] != null && actual[kind].length === 1 && actual[kind][0] === expected[kind]);
|
|
251
|
+
topologySnapshot = { expected, actual };
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
199
255
|
const errors = [];
|
|
200
256
|
if (!engineOk) {
|
|
201
257
|
const state = control && control.state ? control.state : 'unknown';
|
|
202
258
|
const pidLabel = pid || 'none';
|
|
203
|
-
errors.push(`Engine is not healthy (state=${state}, pid=${pidLabel}, alive=${engineAlive ? 'yes' : 'no'})`);
|
|
259
|
+
errors.push(`Engine is not healthy (state=${state}, pid=${pidLabel}, alive=${engineAlive ? 'yes' : 'no'}, owned=${engineOwned ? 'yes' : 'no'})`);
|
|
204
260
|
}
|
|
205
261
|
if (!dashboardOk) {
|
|
206
262
|
const where = dashboardKind === 'http'
|
|
@@ -208,11 +264,19 @@ async function checkRestartHealth(options = {}) {
|
|
|
208
264
|
: `dashboard pid=${normalizePid(dashboardPid) || 'none'} port=${dashboardPort}`;
|
|
209
265
|
errors.push(`Dashboard failed health check at ${where} (${dashboardDetail})`);
|
|
210
266
|
}
|
|
267
|
+
if (!supervisorOk) {
|
|
268
|
+
errors.push(`Supervisor is not healthy (pid=${supervisorSnapshot.pid || 'none'}, alive=${supervisorSnapshot.alive ? 'yes' : 'no'}, recordedPid=${supervisorSnapshot.recordedPid || 'none'}, owned=${supervisorSnapshot.owned ? 'yes' : 'no'})`);
|
|
269
|
+
}
|
|
270
|
+
if (!topologyOk) {
|
|
271
|
+
errors.push(`Minions process topology is not exclusive (${JSON.stringify(topologySnapshot)})`);
|
|
272
|
+
}
|
|
211
273
|
|
|
212
274
|
return {
|
|
213
|
-
ok: engineOk && dashboardOk,
|
|
214
|
-
engine: { state: control && control.state, pid, alive: engineAlive },
|
|
275
|
+
ok: engineOk && dashboardOk && supervisorOk && topologyOk,
|
|
276
|
+
engine: { state: control && control.state, pid, alive: engineAlive, owned: engineOwned },
|
|
215
277
|
dashboard: dashboardSnapshot,
|
|
278
|
+
supervisor: supervisorSnapshot,
|
|
279
|
+
topology: topologySnapshot,
|
|
216
280
|
errors,
|
|
217
281
|
};
|
|
218
282
|
}
|
|
@@ -310,6 +374,7 @@ module.exports = {
|
|
|
310
374
|
isProcessAlive,
|
|
311
375
|
isPortListening,
|
|
312
376
|
readEngineControl,
|
|
377
|
+
readSupervisorPid,
|
|
313
378
|
normalizePid,
|
|
314
379
|
},
|
|
315
380
|
};
|
|
@@ -838,6 +838,12 @@ const capabilities = {
|
|
|
838
838
|
budgetCap: true,
|
|
839
839
|
// Supports `--bare` (suppress CLAUDE.md auto-discovery)
|
|
840
840
|
bareMode: true,
|
|
841
|
+
// The Claude Code CLI reads repo-authored CLAUDE.md files itself at startup
|
|
842
|
+
// (unless `--bare`). TRUE here tells the engine's CLAUDE.md-propagation layer
|
|
843
|
+
// (engine/claude-md-context.js, injected in engine/playbook.js) to SKIP this
|
|
844
|
+
// runtime — re-injecting CLAUDE.md content into the prompt would double it.
|
|
845
|
+
// Copilot/Codex set this false because they have no native CLAUDE.md auto-load.
|
|
846
|
+
claudeMdNativeDiscovery: true,
|
|
841
847
|
// Supports `--fallback-model <id>`
|
|
842
848
|
fallbackModel: true,
|
|
843
849
|
// Engine controls session persistence (writes session.json on completion)
|
package/engine/runtimes/codex.js
CHANGED
|
@@ -840,6 +840,20 @@ function getModelCacheKey({ env = process.env, timeoutMs = 10000 } = {}) {
|
|
|
840
840
|
}
|
|
841
841
|
}
|
|
842
842
|
|
|
843
|
+
function getModelCacheKey({ env = process.env, timeoutMs = 10000 } = {}) {
|
|
844
|
+
const resolved = resolveBinary({ env });
|
|
845
|
+
if (!resolved) return null;
|
|
846
|
+
try {
|
|
847
|
+
return _execFileCapture(resolved.bin, [...(resolved.leadingArgs || []), '--version'], {
|
|
848
|
+
env,
|
|
849
|
+
timeoutMs,
|
|
850
|
+
native: resolved.native,
|
|
851
|
+
}).trim() || null;
|
|
852
|
+
} catch {
|
|
853
|
+
return null;
|
|
854
|
+
}
|
|
855
|
+
}
|
|
856
|
+
|
|
843
857
|
function createStreamConsumer(ctx) {
|
|
844
858
|
let deltaText = '';
|
|
845
859
|
let terminalText = '';
|
|
@@ -901,6 +915,10 @@ const capabilities = {
|
|
|
901
915
|
budgetCap: false,
|
|
902
916
|
bareMode: false,
|
|
903
917
|
fallbackModel: false,
|
|
918
|
+
// Codex has no native CLAUDE.md auto-load — FALSE opts it into the engine's
|
|
919
|
+
// CLAUDE.md-propagation layer (engine/claude-md-context.js) so repo-authored
|
|
920
|
+
// CLAUDE.md guidance reaches the agent prompt.
|
|
921
|
+
claudeMdNativeDiscovery: false,
|
|
904
922
|
sessionPersistenceControl: false,
|
|
905
923
|
resumePromptCarryover: true,
|
|
906
924
|
streamConsumer: true,
|
|
@@ -1273,6 +1273,13 @@ const capabilities = {
|
|
|
1273
1273
|
budgetCap: false,
|
|
1274
1274
|
// No --bare equivalent.
|
|
1275
1275
|
bareMode: false,
|
|
1276
|
+
// Copilot has NO native CLAUDE.md auto-load — it only reads its own AGENTS.md
|
|
1277
|
+
// (native custom-instruction discovery owned by the CLI itself). FALSE
|
|
1278
|
+
// here opts this runtime INTO the engine's CLAUDE.md-propagation layer
|
|
1279
|
+
// (engine/claude-md-context.js) so repo-authored CLAUDE.md guidance is
|
|
1280
|
+
// surfaced in the prompt. This is orthogonal to AGENTS.md discovery —
|
|
1281
|
+
// CLAUDE.md carries no split-brain risk because Copilot never reads it itself.
|
|
1282
|
+
claudeMdNativeDiscovery: false,
|
|
1276
1283
|
// No --fallback-model
|
|
1277
1284
|
fallbackModel: false,
|
|
1278
1285
|
// Copilot manages session state internally in ~/.copilot/session-state/
|
package/engine/shared.js
CHANGED
|
@@ -1037,8 +1037,16 @@ function validateDispatchTmpDir(dirPath) {
|
|
|
1037
1037
|
|
|
1038
1038
|
function removeDispatchTmpDir(dirPath) {
|
|
1039
1039
|
if (!validateDispatchTmpDir(dirPath)) return false;
|
|
1040
|
-
try {
|
|
1041
|
-
|
|
1040
|
+
try {
|
|
1041
|
+
_retryFsOp(
|
|
1042
|
+
() => fs.rmSync(dirPath, { recursive: true, force: true }),
|
|
1043
|
+
`removeDispatchTmpDir(${dirPath})`
|
|
1044
|
+
);
|
|
1045
|
+
return true;
|
|
1046
|
+
} catch (e) {
|
|
1047
|
+
log('warn', `removeDispatchTmpDir failed after retries: ${e.message}`, { path: dirPath, error: e.code });
|
|
1048
|
+
return false;
|
|
1049
|
+
}
|
|
1042
1050
|
}
|
|
1043
1051
|
|
|
1044
1052
|
// Read a file using O_NOFOLLOW where the platform supports it. On Windows
|
|
@@ -1439,48 +1447,37 @@ function withFileLock(lockPath, fn, {
|
|
|
1439
1447
|
const isLockRaceErr = err.code === 'EEXIST' ||
|
|
1440
1448
|
(process.platform === 'win32' && err.code === 'EPERM');
|
|
1441
1449
|
if (!isLockRaceErr) throw err;
|
|
1442
|
-
//
|
|
1443
|
-
//
|
|
1444
|
-
//
|
|
1445
|
-
// - dead PID → reap.
|
|
1446
|
-
// - alive PID, mtime <= 5× → don't reap (legitimate slow holder).
|
|
1447
|
-
// - alive PID, mtime > 5× → reap (last-resort guard against
|
|
1448
|
-
// stuck holders that never released).
|
|
1449
|
-
// 3. Legacy / empty / non-JSON lockfile → mtime-only path (reap).
|
|
1450
|
+
// PID-owned locks are safe to reap as soon as their owner is dead.
|
|
1451
|
+
// Alive owners retain the 5× stale window; legacy/partial lockfiles
|
|
1452
|
+
// without a readable PID retain the mtime-only stale window.
|
|
1450
1453
|
try {
|
|
1451
1454
|
const stat = fs.statSync(lockPath);
|
|
1452
1455
|
const mtimeAge = Date.now() - stat.mtimeMs;
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
holderPid = parsed.pid;
|
|
1460
|
-
}
|
|
1461
|
-
} catch { /* legacy/empty/corrupt lock → fall through to mtime-only */ }
|
|
1462
|
-
|
|
1463
|
-
let shouldReap;
|
|
1464
|
-
if (holderPid !== null) {
|
|
1465
|
-
shouldReap = !processUtils.isPidAlive(holderPid) || mtimeAge > LOCK_STALE_MS * 5;
|
|
1466
|
-
} else {
|
|
1467
|
-
// Legacy empty/non-JSON lockfile: trust mtime alone
|
|
1468
|
-
shouldReap = true;
|
|
1456
|
+
let holderPid = null;
|
|
1457
|
+
try {
|
|
1458
|
+
const raw = fs.readFileSync(lockPath, 'utf8');
|
|
1459
|
+
const parsed = JSON.parse(raw);
|
|
1460
|
+
if (parsed && Number.isFinite(parsed.pid) && parsed.pid > 0) {
|
|
1461
|
+
holderPid = parsed.pid;
|
|
1469
1462
|
}
|
|
1463
|
+
} catch { /* legacy/empty/corrupt lock → mtime-only fallback */ }
|
|
1470
1464
|
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1482
|
-
|
|
1465
|
+
const shouldReap = holderPid !== null
|
|
1466
|
+
? !processUtils.isPidAlive(holderPid) || mtimeAge > LOCK_STALE_MS * 5
|
|
1467
|
+
: mtimeAge > LOCK_STALE_MS;
|
|
1468
|
+
|
|
1469
|
+
if (shouldReap) {
|
|
1470
|
+
try {
|
|
1471
|
+
fs.unlinkSync(lockPath);
|
|
1472
|
+
} catch (unlinkErr) {
|
|
1473
|
+
// ENOENT: another process deleted the lock between stat and unlink — safe to retry.
|
|
1474
|
+
// EPERM on Windows: file is in 'pending delete' state from a concurrent
|
|
1475
|
+
// reaper — the OS will finalize the delete shortly; retry will succeed.
|
|
1476
|
+
const tolerable = unlinkErr.code === 'ENOENT' ||
|
|
1477
|
+
(process.platform === 'win32' && unlinkErr.code === 'EPERM');
|
|
1478
|
+
if (!tolerable) throw unlinkErr;
|
|
1483
1479
|
}
|
|
1480
|
+
continue; // lock just removed — retry immediately
|
|
1484
1481
|
}
|
|
1485
1482
|
} catch (staleErr) {
|
|
1486
1483
|
// ENOENT from statSync: lock file disappeared between EEXIST and stat — retry will succeed.
|
|
@@ -3120,9 +3117,8 @@ const ENGINE_DEFAULTS = {
|
|
|
3120
3117
|
// description the validator will never accept (see stage-output
|
|
3121
3118
|
// substitution bug above) got re-evaluated forever, once per tick,
|
|
3122
3119
|
// invisible to the dashboard. Once the SAME (unchanged) description is
|
|
3123
|
-
// rejected this many consecutive times, the WI
|
|
3124
|
-
//
|
|
3125
|
-
// re-queued into review again.
|
|
3120
|
+
// rejected this many consecutive times, the WI stays pending but is marked
|
|
3121
|
+
// exhausted so discovery pauses until an operator edits it.
|
|
3126
3122
|
preDispatchEvalMaxRejections: 5,
|
|
3127
3123
|
completionNonceRequired: false, // P-d2a8f6c1 (agent trust boundary F8): when true, a missing `nonce` field in the completion JSON hard-fails the dispatch with failure_class:'completion-nonce-mismatch'. Default false for one release so older agents/runtime caches that haven't picked up the prompt change degrade with a warning instead of breaking. Mismatched nonces hard-fail regardless of this flag. See docs/completion-reports.md → "Trust boundary".
|
|
3128
3124
|
autoApplyReviewVote: false, // Master gate for platform vote mutations; local verdicts remain informational when false.
|
|
@@ -3255,6 +3251,7 @@ const ENGINE_DEFAULTS = {
|
|
|
3255
3251
|
maxMeetingPromptBytes: 16 * 1024, // cap meeting findings/debate context injected into prompts
|
|
3256
3252
|
maxMeetingHumanNotesBytes: 2 * 1024, // cap human note bullet lists injected into meeting prompts
|
|
3257
3253
|
maxPipelineMeetingContextBytes: 16 * 1024, // cap aggregated meeting/dependency context for pipeline plan generation
|
|
3254
|
+
maxPipelineStageHandoffBytes: 32 * 1024, // cap current-run note content exposed through {{stages.<id>.handoff}}
|
|
3258
3255
|
notesArchiveMaxFiles: 2000, // keep notes/archive bounded during periodic cleanup
|
|
3259
3256
|
// ── Worktree pool (W-mp73ya3e000me6c5) ─────────────────────────────────────
|
|
3260
3257
|
// Per-project warm pool: completed worktrees are parked detached at
|
|
@@ -3324,7 +3321,7 @@ const ENGINE_DEFAULTS = {
|
|
|
3324
3321
|
'node', 'bun', 'npm', 'npx', 'pnpm', 'yarn',
|
|
3325
3322
|
'python', 'python3', 'pip', 'pip3',
|
|
3326
3323
|
'docker', 'podman',
|
|
3327
|
-
'adb', 'emulator',
|
|
3324
|
+
'adb', 'emulator', 'maestro',
|
|
3328
3325
|
'gradle', 'gradlew', 'mvn',
|
|
3329
3326
|
'pwsh', 'powershell', 'bash', 'sh',
|
|
3330
3327
|
'curl', 'wget',
|
|
@@ -3629,6 +3626,28 @@ function resolveAgentBareMode(agent, engine) {
|
|
|
3629
3626
|
return false;
|
|
3630
3627
|
}
|
|
3631
3628
|
|
|
3629
|
+
// W-mrdon0pe000l045a — resolve whether repo-authored CLAUDE.md instructions
|
|
3630
|
+
// should be propagated into the prompt for runtimes that don't auto-discover
|
|
3631
|
+
// CLAUDE.md natively (Copilot/Codex). Precedence mirrors
|
|
3632
|
+
// `resolveLiveCheckoutAutoReset`:
|
|
3633
|
+
// 1. per-project boolean `project.propagateClaudeMdForNonClaudeRuntimes`
|
|
3634
|
+
// 2. fleet-wide boolean `engine.propagateClaudeMdForNonClaudeRuntimes`
|
|
3635
|
+
// 3. hardcoded default TRUE (additive, on by default)
|
|
3636
|
+
// Only an explicit boolean counts as "set" at either level — undefined/null/
|
|
3637
|
+
// string are treated as unset so the next tier (or the TRUE default) decides.
|
|
3638
|
+
// This gate is orthogonal to each runtime's native AGENTS.md / repo-instruction
|
|
3639
|
+
// discovery by design (see ENGINE_DEFAULTS.propagateClaudeMdForNonClaudeRuntimes).
|
|
3640
|
+
// Pure, no I/O.
|
|
3641
|
+
function resolvePropagateClaudeMdForNonClaudeRuntimes(project, engine) {
|
|
3642
|
+
if (project && typeof project === 'object' && typeof project.propagateClaudeMdForNonClaudeRuntimes === 'boolean') {
|
|
3643
|
+
return project.propagateClaudeMdForNonClaudeRuntimes;
|
|
3644
|
+
}
|
|
3645
|
+
if (engine && typeof engine === 'object' && typeof engine.propagateClaudeMdForNonClaudeRuntimes === 'boolean') {
|
|
3646
|
+
return engine.propagateClaudeMdForNonClaudeRuntimes;
|
|
3647
|
+
}
|
|
3648
|
+
return true;
|
|
3649
|
+
}
|
|
3650
|
+
|
|
3632
3651
|
// ─── Legacy ccModel → defaultModel Migration ─────────────────────────────────
|
|
3633
3652
|
//
|
|
3634
3653
|
// Pre-P-3b8e5f1d, `engine.ccModel` was the single fleet-wide model knob (it
|
|
@@ -4500,7 +4519,7 @@ const FAILURE_CLASS = {
|
|
|
4500
4519
|
OUTPUT_TRUNCATED: 'output-truncated', // P-8e4c2a17: the agent streamed more stdout than the engine's hard capture cap (engine.js AGENT_OUTPUT_CAP_BYTES, 1MB) BEFORE the terminal `result` event arrived. The result (session id, completion block, final text) lives at the END of the stream, so it fell outside the captured window and parseOutput found nothing — the dispatch would otherwise fail as an opaque UNKNOWN and retry to death. Surfaced loudly with an actionable message; non-retryable (mechanical retry just reproduces the overflow — the agent must reduce output volume or the task must be split).
|
|
4501
4520
|
VERIFY_MISSING_PR: 'verify-missing-pr', // Verify completed without an attached or tracked PR.
|
|
4502
4521
|
PROJECT_NOT_FOUND: 'project_not_found', // W-mqv2wsy30002e090: a work item's `project` field names a project that is not configured (resolveConfiguredProject / formatUnknownProjectError → `Project "<name>" not found. Known projects: …`). Stamped at discovery time so the dashboard PR-column renderer surfaces the red failReason snippet like other non-retryable failures instead of burying it in the Agent column. Non-retryable — operator must fix the WI's project field or configure the project.
|
|
4503
|
-
PRE_DISPATCH_EVAL_STUCK: 'pre-dispatch-eval-stuck', //
|
|
4522
|
+
PRE_DISPATCH_EVAL_STUCK: 'pre-dispatch-eval-stuck', // Legacy persisted classification normalized by migration 025. New exhausted evaluations remain pending with _preDispatchEval.exhausted so they are not counted as agent execution failures.
|
|
4504
4523
|
UNKNOWN: 'unknown', // Unclassified failure
|
|
4505
4524
|
};
|
|
4506
4525
|
|
|
@@ -8491,7 +8510,7 @@ module.exports = {
|
|
|
8491
8510
|
ENGINE_DEFAULTS,
|
|
8492
8511
|
resolvePollFlag, // P-c4d8e1a3 — granular per-poller flag resolution
|
|
8493
8512
|
resolveAgentCli, resolveCcCli, resolveCcUseWorkerPool, resolveAgentUseWorkerPool, resolveAgentAcpPoolSize, resolveAgentModel, resolveCcModel,
|
|
8494
|
-
resolveAgentMaxBudget, resolveAgentBareMode,
|
|
8513
|
+
resolveAgentMaxBudget, resolveAgentBareMode, resolvePropagateClaudeMdForNonClaudeRuntimes,
|
|
8495
8514
|
applyLegacyCcModelMigration, _resetLegacyCcModelMigrationFlag,
|
|
8496
8515
|
runtimeConfigWarnings,
|
|
8497
8516
|
projectWorkSourceWarnings,
|
package/engine/supervisor.js
CHANGED
|
@@ -104,6 +104,8 @@ const SUPERVISOR_STALE_ENGINE_HEARTBEAT_MS = Number(process.env.MINIONS_SUPERVIS
|
|
|
104
104
|
// 3 strikes ≈ 90s of continuous unresponsiveness before we act.
|
|
105
105
|
const DASH_HEALTH_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_TIMEOUT_MS) || 5000;
|
|
106
106
|
const DASH_HEALTH_MAX_FAILS = Number(process.env.MINIONS_SUPERVISOR_DASH_HEALTH_FAILS) || 3;
|
|
107
|
+
const REAP_WAIT_TIMEOUT_MS = Number(process.env.MINIONS_SUPERVISOR_REAP_TIMEOUT_MS) || 10000;
|
|
108
|
+
const REAP_WAIT_POLL_MS = Number(process.env.MINIONS_SUPERVISOR_REAP_POLL_MS) || 100;
|
|
107
109
|
const isWin = process.platform === 'win32';
|
|
108
110
|
|
|
109
111
|
function safeReadJson(p) {
|
|
@@ -126,11 +128,27 @@ function isStopIntentSet() {
|
|
|
126
128
|
return fs.existsSync(STOP_INTENT_PATH_FN());
|
|
127
129
|
}
|
|
128
130
|
|
|
129
|
-
function
|
|
130
|
-
|
|
131
|
+
function _readPosixProcessState(pid) {
|
|
132
|
+
const result = spawnSync('ps', ['-o', 'stat=', '-p', String(pid)], {
|
|
133
|
+
encoding: 'utf8',
|
|
134
|
+
timeout: 3000,
|
|
135
|
+
windowsHide: true,
|
|
136
|
+
stdio: ['ignore', 'pipe', 'ignore'],
|
|
137
|
+
});
|
|
138
|
+
if (result.error || result.status !== 0) return null;
|
|
139
|
+
return String(result.stdout || '').trim();
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function isPidAlive(pid, {
|
|
143
|
+
platform = process.platform,
|
|
144
|
+
kill = process.kill,
|
|
145
|
+
readPosixProcessState = _readPosixProcessState,
|
|
146
|
+
} = {}) {
|
|
147
|
+
const n = Number(pid);
|
|
148
|
+
if (!Number.isInteger(n) || n <= 0) return false;
|
|
131
149
|
try {
|
|
132
|
-
if (
|
|
133
|
-
const out = execSync(`tasklist /FI "PID eq ${
|
|
150
|
+
if (platform === 'win32') {
|
|
151
|
+
const out = execSync(`tasklist /FI "PID eq ${n}" /NH`, {
|
|
134
152
|
encoding: 'utf8', timeout: 3000, windowsHide: true,
|
|
135
153
|
});
|
|
136
154
|
// Word-boundary + image-name match (mirrors engine/restart-health.js) so a
|
|
@@ -138,12 +156,16 @@ function isPidAlive(pid) {
|
|
|
138
156
|
// alive. Fall back to an inline regex if shared.js isn't loadable.
|
|
139
157
|
const shared = _sharedOrNull();
|
|
140
158
|
if (shared && typeof shared.tasklistOutputShowsPid === 'function') {
|
|
141
|
-
return shared.tasklistOutputShowsPid(out,
|
|
159
|
+
return shared.tasklistOutputShowsPid(out, n, { imageName: 'node' });
|
|
142
160
|
}
|
|
143
|
-
return new RegExp(`\\b${
|
|
161
|
+
return new RegExp(`\\b${n}\\b`).test(out) && out.toLowerCase().includes('node');
|
|
144
162
|
}
|
|
145
|
-
|
|
146
|
-
|
|
163
|
+
kill(n, 0);
|
|
164
|
+
// kill(0) also succeeds for Unix zombies. They cannot execute or retain
|
|
165
|
+
// runtime resources, and a synchronous reap wait prevents Node from
|
|
166
|
+
// processing SIGCHLD, so treating them as live deadlocks every respawn.
|
|
167
|
+
const state = readPosixProcessState(n);
|
|
168
|
+
return state === null || (state !== '' && !state.startsWith('Z'));
|
|
147
169
|
} catch { return false; }
|
|
148
170
|
}
|
|
149
171
|
|
|
@@ -305,14 +327,43 @@ function _killHard(pid) {
|
|
|
305
327
|
} catch { /* already gone */ }
|
|
306
328
|
}
|
|
307
329
|
|
|
330
|
+
function _sleepSync(ms) {
|
|
331
|
+
if (!(ms > 0)) return;
|
|
332
|
+
Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
|
|
333
|
+
}
|
|
334
|
+
|
|
335
|
+
function waitForPidsToExit(pids, {
|
|
336
|
+
timeoutMs = REAP_WAIT_TIMEOUT_MS,
|
|
337
|
+
pollMs = REAP_WAIT_POLL_MS,
|
|
338
|
+
isAlive = isPidAlive,
|
|
339
|
+
sleep = _sleepSync,
|
|
340
|
+
} = {}) {
|
|
341
|
+
const unique = [...new Set((pids || []).map(Number).filter(pid => Number.isInteger(pid) && pid > 0))];
|
|
342
|
+
const deadline = Date.now() + Math.max(0, timeoutMs);
|
|
343
|
+
let remaining = unique.filter(isAlive);
|
|
344
|
+
while (remaining.length > 0 && Date.now() < deadline) {
|
|
345
|
+
sleep(Math.min(pollMs, Math.max(1, deadline - Date.now())));
|
|
346
|
+
remaining = remaining.filter(isAlive);
|
|
347
|
+
}
|
|
348
|
+
return { ok: remaining.length === 0, remaining };
|
|
349
|
+
}
|
|
350
|
+
|
|
308
351
|
// Kill every stray process running `scriptPath`, except this supervisor.
|
|
309
|
-
// Returns the number reaped.
|
|
310
|
-
|
|
352
|
+
// Returns the number reaped. Throws instead of spawning a replacement when any
|
|
353
|
+
// old process survives the bounded wait — binding a fallback port or starting a
|
|
354
|
+
// second engine is worse than retrying on the next supervisor tick.
|
|
355
|
+
function reapStrayProcesses(scriptPath, label, options = {}) {
|
|
311
356
|
if (!REAP_ENABLED) return 0;
|
|
312
|
-
const
|
|
357
|
+
const listPids = options.listPids || listNodePidsMatching;
|
|
358
|
+
const kill = options.kill || _killHard;
|
|
359
|
+
const pids = listPids(scriptPath).filter(p => p !== process.pid);
|
|
313
360
|
if (!pids.length) return 0;
|
|
314
361
|
console.log(`[supervisor] Reaping ${pids.length} stray ${label} process(es) before respawn: ${pids.join(', ')}`);
|
|
315
|
-
for (const pid of pids)
|
|
362
|
+
for (const pid of pids) kill(pid);
|
|
363
|
+
const waited = waitForPidsToExit(pids, options);
|
|
364
|
+
if (!waited.ok) {
|
|
365
|
+
throw new Error(`Refusing to respawn ${label}: stale PID(s) still alive after ${options.timeoutMs || REAP_WAIT_TIMEOUT_MS}ms: ${waited.remaining.join(', ')}`);
|
|
366
|
+
}
|
|
316
367
|
return pids.length;
|
|
317
368
|
}
|
|
318
369
|
|
|
@@ -372,11 +423,15 @@ function spawnEngine() {
|
|
|
372
423
|
return proc.pid;
|
|
373
424
|
}
|
|
374
425
|
|
|
375
|
-
function spawnDashboard() {
|
|
426
|
+
function spawnDashboard(requestedPort = _resolveDashPort()) {
|
|
376
427
|
const out = openAppendFd('dashboard-stdio.log');
|
|
377
428
|
const err = openAppendFd('dashboard-stdio.log');
|
|
378
|
-
const env = {
|
|
379
|
-
|
|
429
|
+
const env = {
|
|
430
|
+
...process.env,
|
|
431
|
+
MINIONS_NO_AUTO_OPEN: '1',
|
|
432
|
+
MINIONS_REQUIRE_DASHBOARD_PORT: '1',
|
|
433
|
+
};
|
|
434
|
+
const proc = spawn(process.execPath, [..._sqliteSpawnFlags(), path.join(MINIONS_DIR, 'dashboard.js'), String(requestedPort)], {
|
|
380
435
|
cwd: MINIONS_DIR,
|
|
381
436
|
stdio: ['ignore', out, err],
|
|
382
437
|
detached: true,
|
|
@@ -533,7 +588,7 @@ async function checkDashboard(now) {
|
|
|
533
588
|
// Critically, for the frozen-but-listening case the reap also KILLS the wedged
|
|
534
589
|
// process still holding the port so spawnDashboard() can bind it.
|
|
535
590
|
reapStrayProcesses(path.join(MINIONS_DIR, 'dashboard.js'), 'dashboard');
|
|
536
|
-
const newPid = spawnDashboard();
|
|
591
|
+
const newPid = spawnDashboard(dashPort);
|
|
537
592
|
_lastDashboardRespawnAt = now;
|
|
538
593
|
_dashHealthFailStreak = 0;
|
|
539
594
|
console.log(`[supervisor] Dashboard respawned (new PID: ${newPid})`);
|
|
@@ -647,6 +702,7 @@ module.exports = {
|
|
|
647
702
|
listeningPidsForPort,
|
|
648
703
|
listNodePidsMatching,
|
|
649
704
|
reapStrayProcesses,
|
|
705
|
+
waitForPidsToExit,
|
|
650
706
|
_cmdRunsScript,
|
|
651
707
|
openAppendFd,
|
|
652
708
|
checkEngine,
|
package/engine/timeout.js
CHANGED
|
@@ -23,6 +23,60 @@ const MINIONS_DIR = shared.MINIONS_DIR;
|
|
|
23
23
|
// Only needed for engine().activeProcesses and engine().engineRestartGraceUntil — log/ts come from shared.js
|
|
24
24
|
let _engine = null;
|
|
25
25
|
function engine() { if (!_engine) _engine = require('../engine'); return _engine; }
|
|
26
|
+
const _pendingBranchPreservations = new Set();
|
|
27
|
+
|
|
28
|
+
function resolveDispatchBranchPath(item) {
|
|
29
|
+
if (item?.worktreePath) return item.worktreePath;
|
|
30
|
+
if (item?.originalRef && item.meta?.project?.localPath) return item.meta.project.localPath;
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function preserveDispatchBranch(item, config) {
|
|
35
|
+
const branchPath = resolveDispatchBranchPath(item);
|
|
36
|
+
if (!branchPath || !item.meta?.branch) {
|
|
37
|
+
return Promise.resolve({ pushed: false, reason: 'invalid-worktree' });
|
|
38
|
+
}
|
|
39
|
+
const persistedProject = item.meta?.project || {};
|
|
40
|
+
const project = getProjects(config).find(p => p.name === persistedProject.name) || persistedProject;
|
|
41
|
+
if (!project.localPath) return Promise.resolve({ pushed: false, reason: 'invalid-project' });
|
|
42
|
+
return engine().pushCleanAheadBranch({
|
|
43
|
+
rootDir: project.localPath,
|
|
44
|
+
worktreePath: branchPath,
|
|
45
|
+
branchName: item.meta.branch,
|
|
46
|
+
mainBranch: shared.resolveMainBranch(project.localPath, project.mainBranch),
|
|
47
|
+
project,
|
|
48
|
+
gitOpts: { env: shared.gitEnv() },
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async function _finalizeDeadDispatch(item, reason, failureClass, config, deps = {}) {
|
|
53
|
+
const preserveBranch = deps.preserveBranch || preserveDispatchBranch;
|
|
54
|
+
const restoreLiveCheckout = deps.restoreLiveCheckout ||
|
|
55
|
+
require('./live-checkout').maybeRestoreLiveCheckoutFromRecord;
|
|
56
|
+
const complete = deps.complete || dispatch().completeDispatch;
|
|
57
|
+
|
|
58
|
+
if (resolveDispatchBranchPath(item) && item.meta?.branch) {
|
|
59
|
+
try {
|
|
60
|
+
await preserveBranch(item, config);
|
|
61
|
+
} catch (err) {
|
|
62
|
+
log('warn', `Orphan branch preservation failed for ${item.id}: ${err.message}`);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
if (item.originalRef && item.meta?.branch && item.meta?.project?.localPath) {
|
|
66
|
+
try {
|
|
67
|
+
await restoreLiveCheckout({
|
|
68
|
+
item,
|
|
69
|
+
isTerminalFailure: true,
|
|
70
|
+
resultLabel: 'orphaned',
|
|
71
|
+
log: (lvl, msg) => log(lvl, msg),
|
|
72
|
+
writeInboxAlert: dispatch().writeInboxAlert,
|
|
73
|
+
});
|
|
74
|
+
} catch (err) {
|
|
75
|
+
log('warn', 'live-checkout restore (orphan): ' + err.message);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
complete(item.id, DISPATCH_RESULT.ERROR, reason, '', failureClass ? { failureClass } : {});
|
|
79
|
+
}
|
|
26
80
|
|
|
27
81
|
// Lazy require for dispatch module (also circular via engine)
|
|
28
82
|
let _dispatch = null;
|
|
@@ -664,7 +718,27 @@ function checkTimeouts(config) {
|
|
|
664
718
|
const deadItems = [];
|
|
665
719
|
const legacyAnnotationClears = new Set();
|
|
666
720
|
|
|
667
|
-
function completeFromOutput(item, liveLogPath, processExitCode, detectedLogText, hasProcess) {
|
|
721
|
+
function completeFromOutput(item, liveLogPath, processExitCode, detectedLogText, hasProcess, preservationDone = false, expectedProcInfo = activeProcesses.get(item.id)) {
|
|
722
|
+
if (!preservationDone && resolveDispatchBranchPath(item) && item.meta?.branch) {
|
|
723
|
+
if (_pendingBranchPreservations.has(item.id)) return;
|
|
724
|
+
_pendingBranchPreservations.add(item.id);
|
|
725
|
+
preserveDispatchBranch(item, config)
|
|
726
|
+
.catch(err => log('warn', `Output-completion branch preservation failed for ${item.id}: ${err.message}`))
|
|
727
|
+
.then(() => completeFromOutput(item, liveLogPath, processExitCode, detectedLogText, hasProcess, true, expectedProcInfo))
|
|
728
|
+
.finally(() => _pendingBranchPreservations.delete(item.id));
|
|
729
|
+
return;
|
|
730
|
+
}
|
|
731
|
+
if (!(getDispatch().active || []).some(active => active.id === item.id)) return;
|
|
732
|
+
if (hasProcess) {
|
|
733
|
+
const currentProcInfo = activeProcesses.get(item.id);
|
|
734
|
+
if (currentProcInfo !== expectedProcInfo ||
|
|
735
|
+
currentProcInfo?._steeringAt ||
|
|
736
|
+
currentProcInfo?._steeringMessage ||
|
|
737
|
+
currentProcInfo?._steeringNoSession) {
|
|
738
|
+
log('info', `${item.id}: output-detected completion yielded to the close/steering owner after branch preservation`);
|
|
739
|
+
return;
|
|
740
|
+
}
|
|
741
|
+
}
|
|
668
742
|
const isSuccess = processExitCode === 0;
|
|
669
743
|
log('info', `Agent ${item.agent} (${item.id}) completed via output detection (exit code ${processExitCode}, ${isSuccess ? 'success' : 'error'})`);
|
|
670
744
|
|
|
@@ -956,22 +1030,18 @@ function checkTimeouts(config) {
|
|
|
956
1030
|
|
|
957
1031
|
// Clean up dead items
|
|
958
1032
|
for (const { item, reason, failureClass } of deadItems) {
|
|
959
|
-
|
|
960
|
-
//
|
|
961
|
-
//
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
resultLabel: 'orphaned',
|
|
970
|
-
log: (lvl, msg) => log(lvl, msg),
|
|
971
|
-
writeInboxAlert: dispatch().writeInboxAlert,
|
|
972
|
-
}).catch(() => {});
|
|
973
|
-
} catch (e) { log('warn', 'live-checkout restore (orphan): ' + e.message); }
|
|
1033
|
+
// #853 — stale orphans have no process close event, so preserve their
|
|
1034
|
+
// clean committed branch explicitly before completeDispatch makes the
|
|
1035
|
+
// worktree eligible for teardown/quarantine.
|
|
1036
|
+
if (resolveDispatchBranchPath(item) && item.meta?.branch) {
|
|
1037
|
+
if (_pendingBranchPreservations.has(item.id)) continue;
|
|
1038
|
+
_pendingBranchPreservations.add(item.id);
|
|
1039
|
+
_finalizeDeadDispatch(item, reason, failureClass, config, { complete: completeDispatch })
|
|
1040
|
+
.catch(err => log('warn', `Orphan cleanup failed for ${item.id}: ${err.message}`))
|
|
1041
|
+
.finally(() => _pendingBranchPreservations.delete(item.id));
|
|
1042
|
+
continue;
|
|
974
1043
|
}
|
|
1044
|
+
completeDispatch(item.id, DISPATCH_RESULT.ERROR, reason, '', failureClass ? { failureClass } : {});
|
|
975
1045
|
}
|
|
976
1046
|
|
|
977
1047
|
// Clear legacy blocking-tool annotations; process liveness no longer depends on tool parsing.
|
|
@@ -1055,4 +1125,5 @@ module.exports = {
|
|
|
1055
1125
|
// exported for testing — steering clamp helpers and deferred checkpoint
|
|
1056
1126
|
_clampSteeringDeferredMaxMs, _clampSteeringMaxKillRetries, _appendLiveOutputLine, deferSteeringUntilCheckpoint,
|
|
1057
1127
|
killTrackedProcess, _escalatePlatformKill, // exported for testing — P-6e2a4c15 cancel-vs-kill
|
|
1128
|
+
_finalizeDeadDispatch, // exported for testing
|
|
1058
1129
|
};
|
|
@@ -113,7 +113,8 @@ function buildSource(kind, parts) {
|
|
|
113
113
|
}
|
|
114
114
|
|
|
115
115
|
if (k === 'pinned-note' || k === 'team-notes' || k === 'agent-memory'
|
|
116
|
-
|| k === 'wi-reference' || k === 'doc-content' || k === 'doc-selection'
|
|
116
|
+
|| k === 'wi-reference' || k === 'doc-content' || k === 'doc-selection'
|
|
117
|
+
|| k === 'project-instructions') {
|
|
117
118
|
return parts.path ? `${k}:${get('path')}` : k;
|
|
118
119
|
}
|
|
119
120
|
if (k === 'inbox') {
|