@yemi33/minions 0.1.2144 → 0.1.2146

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,458 @@
1
+ // engine/watchdog.js — out-of-process recovery scheduled by the OS.
2
+ //
3
+ // PROBLEM (cluster-kill):
4
+ // The in-process supervisor (engine/supervisor.js) respawns engine/dashboard
5
+ // when one of them dies in isolation, but it can't respawn ITSELF. When
6
+ // something kills all three at once — Windows job-object teardown when a
7
+ // parent terminal closes (RDP disconnect, Windows Terminal auto-update,
8
+ // manual close), Linux OOM-killer, systemd cgroup memory limits, manual
9
+ // `pkill -f node`, system reboot — recovery requires an external agent.
10
+ //
11
+ // SOLUTION:
12
+ // Register `minions watchdog tick` with the OS scheduler (Task Scheduler /
13
+ // launchd / systemd --user) to run every N minutes. Each tick probes the
14
+ // stack, logs the verdict, and invokes `minions start`/`restart` if dead.
15
+ // The scheduler itself lives outside any user terminal's job/session tree,
16
+ // so it survives every cluster-kill scenario that takes minions down.
17
+ //
18
+ // CONTRACT:
19
+ // tick(opts) → always resolves; never throws to the caller.
20
+ // The scheduler must never see a non-zero exit,
21
+ // or it may start escalating (eventvwr, syslog,
22
+ // email-on-failure) for the very recovery we're
23
+ // trying to make boring.
24
+ // install(opts) → throws on platform-tool failure (so the user
25
+ // sees it; install is interactive).
26
+ // uninstall() → idempotent; removing a non-existent
27
+ // registration is success, not error.
28
+ // status() → returns { installed, scheduler, … details }.
29
+ //
30
+ // Cross-platform note: install/uninstall/status branch on process.platform.
31
+ // Each platform's helper is self-contained so unit tests can stub spawnSync.
32
+
33
+ const fs = require('fs');
34
+ const path = require('path');
35
+ const os = require('os');
36
+ const { spawn, spawnSync } = require('child_process');
37
+
38
+ const DEFAULT_INTERVAL_MIN = 5;
39
+ const DEFAULT_DASH_PORT = 7331;
40
+ const WIN_TASK_NAME = 'MinionsWatchdog';
41
+ const MAC_PLIST_LABEL = 'io.minions.watchdog';
42
+ const LINUX_UNIT_NAME = 'minions-watchdog';
43
+ const TICK_SPAWN_TIMEOUT_MS = 120000;
44
+ const WATCHDOG_LOG_NAME = 'watchdog-stdio.log';
45
+ // Cap the watchdog log so a misbehaving scheduler can't fill the disk
46
+ // over months. Truncates to half-size on overflow (oldest lines dropped).
47
+ const WATCHDOG_LOG_MAX_BYTES = 2 * 1024 * 1024;
48
+
49
+ function logLine(minionsHome, msg) {
50
+ try {
51
+ const dir = path.join(minionsHome, 'engine');
52
+ fs.mkdirSync(dir, { recursive: true });
53
+ const logPath = path.join(dir, WATCHDOG_LOG_NAME);
54
+ try {
55
+ const stat = fs.statSync(logPath);
56
+ if (stat.size > WATCHDOG_LOG_MAX_BYTES) {
57
+ const buf = fs.readFileSync(logPath);
58
+ const tail = buf.subarray(Math.floor(buf.length / 2));
59
+ const firstNewline = tail.indexOf(0x0a);
60
+ const trimmed = firstNewline >= 0 ? tail.subarray(firstNewline + 1) : tail;
61
+ fs.writeFileSync(logPath, trimmed);
62
+ }
63
+ } catch { /* file may not exist yet — that's fine */ }
64
+ fs.appendFileSync(logPath, `[${new Date().toISOString()}] ${msg}\n`);
65
+ } catch { /* logging failure must never escalate to the scheduler */ }
66
+ }
67
+
68
+ function isPidAlive(pid) {
69
+ if (!pid || !Number.isInteger(pid) || pid <= 0) return false;
70
+ try { process.kill(pid, 0); return true; } catch { return false; }
71
+ }
72
+
73
+ /**
74
+ * Probe engine + dashboard and recover if needed. Always resolves (never
75
+ * throws). Returns { healthy, action, exitCode? } describing what was done.
76
+ *
77
+ * Recovery policy:
78
+ * - stop-intent flag set → STAND DOWN (user wanted minions stopped;
79
+ * respawning would undo `minions stop`)
80
+ * - both alive → no-op
81
+ * - both dead → `minions start` (idempotent fast path)
82
+ * - one alive one dead → `minions restart` (only restart can
83
+ * reconcile partial state — start refuses
84
+ * by design)
85
+ */
86
+ async function tick(opts) {
87
+ opts = opts || {};
88
+ const minionsHome = opts.minionsHome;
89
+ const minionsBin = opts.minionsBin;
90
+ const dashPort = opts.dashPort || DEFAULT_DASH_PORT;
91
+ const readEnginePid = opts.readEnginePid;
92
+ const isPortListening = opts.isPortListening;
93
+ const isStopIntentSet = opts.isStopIntentSet;
94
+ const spawner = opts.spawner || spawn;
95
+ const now = opts.now || (() => new Date());
96
+
97
+ if (!minionsHome || !minionsBin || !readEnginePid || !isPortListening) {
98
+ // Programmer error — caller forgot a dep. Try to log when minionsHome is
99
+ // available (logLine swallows errors of its own), then resolve with a
100
+ // marker. The scheduler must never see a non-zero exit.
101
+ if (minionsHome) logLine(minionsHome, 'missing-deps → noop (caller did not provide all required helpers)');
102
+ return { healthy: false, action: 'noop', error: 'missing-deps' };
103
+ }
104
+
105
+ // Respect `minions stop` / `minions uninstall` / mid-restart windows.
106
+ // On forks that ship an in-process supervisor (yemi33/minions), this flag
107
+ // is the same one the supervisor uses to suppress engine respawns; the
108
+ // watchdog MUST honor it too, or it would brain-deadly restart minions
109
+ // every N minutes after the user explicitly stopped it.
110
+ // On forks that DON'T ship a supervisor + stop-intent producer (e.g.
111
+ // opg-microsoft/minions), `isStopIntentSet` is intentionally absent — we
112
+ // treat that as "no stop-intent system, proceed with recovery" (fail-open).
113
+ let stopWanted = false;
114
+ if (typeof isStopIntentSet === 'function') {
115
+ try { stopWanted = !!isStopIntentSet(); } catch { stopWanted = false; }
116
+ }
117
+ if (stopWanted) {
118
+ logLine(minionsHome, `stop-intent set → standing down (no probe, no recovery)`);
119
+ return { healthy: false, action: 'stand-down' };
120
+ }
121
+
122
+ let enginePid = null;
123
+ try { enginePid = readEnginePid(minionsHome); } catch { enginePid = null; }
124
+ const engineAlive = isPidAlive(enginePid);
125
+ let dashUp = false;
126
+ try { dashUp = !!isPortListening(dashPort); } catch { dashUp = false; }
127
+
128
+ if (engineAlive && dashUp) {
129
+ logLine(minionsHome, `ok engine=${enginePid} port=${dashPort} ts=${now().toISOString()}`);
130
+ return { healthy: true, action: 'none' };
131
+ }
132
+
133
+ const partial = engineAlive || dashUp;
134
+ const action = partial ? 'restart' : 'start';
135
+ logLine(
136
+ minionsHome,
137
+ `unhealthy engine=${enginePid || '-'}(${engineAlive ? 'alive' : 'dead'}) ` +
138
+ `port=${dashPort}(${dashUp ? 'up' : 'down'}) → invoking minions ${action}`
139
+ );
140
+
141
+ try {
142
+ // Detach the child so its lifetime is independent of this tick process.
143
+ // The OS scheduler closes our stdio shortly after exit; we don't want
144
+ // that closure to cascade into the new daemon stack.
145
+ const child = spawner(process.execPath, [minionsBin, action], {
146
+ detached: true,
147
+ stdio: 'ignore',
148
+ windowsHide: true,
149
+ });
150
+ if (child && typeof child.unref === 'function') child.unref();
151
+ logLine(minionsHome, `spawned minions ${action} pid=${child && child.pid} (detached)`);
152
+ return { healthy: false, action, spawnedPid: child && child.pid };
153
+ } catch (err) {
154
+ logLine(minionsHome, `FAILED to spawn minions ${action}: ${err && err.message || err}`);
155
+ return { healthy: false, action, error: String(err && err.message || err) };
156
+ }
157
+ }
158
+ // Documented ceiling for any future synchronous variant of tick. Not used
159
+ // by the detached spawn path, but kept reachable so callers can opt in.
160
+ tick.TICK_SPAWN_TIMEOUT_MS = TICK_SPAWN_TIMEOUT_MS;
161
+
162
+ // ─── Install / uninstall / status ────────────────────────────────────────────
163
+
164
+ function install(opts) {
165
+ opts = opts || {};
166
+ const plat = opts.platform || process.platform;
167
+ const minionsBin = opts.minionsBin;
168
+ const intervalMin = Number.isFinite(opts.intervalMin) && opts.intervalMin > 0
169
+ ? Math.floor(opts.intervalMin)
170
+ : DEFAULT_INTERVAL_MIN;
171
+ const nodeBin = opts.nodeBin || process.execPath;
172
+ if (!minionsBin) throw new Error('install: minionsBin is required');
173
+
174
+ if (plat === 'win32') return installWindows({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
175
+ if (plat === 'darwin') return installMac({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
176
+ if (plat === 'linux') return installLinux({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
177
+ throw new Error(`watchdog install: unsupported platform "${plat}"`);
178
+ }
179
+
180
+ function uninstall(opts) {
181
+ opts = opts || {};
182
+ const plat = opts.platform || process.platform;
183
+ if (plat === 'win32') return uninstallWindows({ minionsHome: opts.minionsHome, spawner: opts.spawner });
184
+ if (plat === 'darwin') return uninstallMac({ spawner: opts.spawner });
185
+ if (plat === 'linux') return uninstallLinux({ spawner: opts.spawner });
186
+ throw new Error(`watchdog uninstall: unsupported platform "${plat}"`);
187
+ }
188
+
189
+ function status(opts) {
190
+ opts = opts || {};
191
+ const plat = opts.platform || process.platform;
192
+ if (plat === 'win32') return statusWindows({ spawner: opts.spawner });
193
+ if (plat === 'darwin') return statusMac({ spawner: opts.spawner });
194
+ if (plat === 'linux') return statusLinux({ spawner: opts.spawner });
195
+ throw new Error(`watchdog status: unsupported platform "${plat}"`);
196
+ }
197
+
198
+ // ─── Windows: Task Scheduler ─────────────────────────────────────────────────
199
+
200
+ // On Windows we generate a launcher .cmd in MINIONS_HOME/engine/ that sets
201
+ // MINIONS_HOME before invoking node, then point schtasks at that file. This
202
+ // avoids two failure modes that the obvious schtasks /TR "node bin tick"
203
+ // form has:
204
+ // (a) The task runs without MINIONS_HOME in env (Task Scheduler doesn't
205
+ // inherit the install-time user session env), so a user with a
206
+ // non-default MINIONS_HOME would have the watchdog probe and "recover"
207
+ // the WRONG stack (~/.minions instead of their real install).
208
+ // (b) /TR has hard length + quoting limits for nested arguments. A
209
+ // launcher script keeps schtasks's /TR simple and gives the user a
210
+ // readable, auditable file to inspect on disk.
211
+ function buildWindowsLauncher(opts) {
212
+ return `@echo off\r
213
+ REM Minions watchdog launcher — auto-generated by 'minions watchdog install'.\r
214
+ REM Re-run 'minions watchdog install' to regenerate after moving MINIONS_HOME.\r
215
+ set "MINIONS_HOME=${opts.minionsHome}"\r
216
+ "${opts.nodeBin}" "${opts.minionsBin}" watchdog tick\r
217
+ `;
218
+ }
219
+
220
+ function windowsLauncherPath(minionsHome) {
221
+ return path.join(minionsHome, 'engine', 'watchdog-launcher.cmd');
222
+ }
223
+
224
+ function installWindows(opts) {
225
+ const sp = opts.spawner || spawnSync;
226
+ if (!opts.minionsHome) throw new Error('install (win32): minionsHome is required for launcher path');
227
+ const launcherPath = windowsLauncherPath(opts.minionsHome);
228
+ fs.mkdirSync(path.dirname(launcherPath), { recursive: true });
229
+ fs.writeFileSync(launcherPath, buildWindowsLauncher(opts));
230
+ // /TR points at the launcher. Quote it so spaces in MINIONS_HOME survive.
231
+ const tr = `"${launcherPath}"`;
232
+ const args = [
233
+ '/Create', '/F',
234
+ '/SC', 'MINUTE', '/MO', String(opts.intervalMin),
235
+ '/TN', WIN_TASK_NAME,
236
+ '/TR', tr,
237
+ '/RL', 'LIMITED',
238
+ ];
239
+ const r = sp('schtasks.exe', args, { encoding: 'utf8', windowsHide: true });
240
+ if (r.status !== 0) {
241
+ throw new Error(`schtasks /Create failed (exit ${r.status}): ${(r.stderr || r.stdout || '').trim()}`);
242
+ }
243
+ return { ok: true, scheduler: 'Task Scheduler', taskName: WIN_TASK_NAME, intervalMin: opts.intervalMin, launcherPath };
244
+ }
245
+
246
+ function uninstallWindows(opts) {
247
+ const sp = opts.spawner || spawnSync;
248
+ const r = sp('schtasks.exe', ['/Delete', '/F', '/TN', WIN_TASK_NAME], { encoding: 'utf8', windowsHide: true });
249
+ // Exit 1 from schtasks usually means "task not found" — idempotent success.
250
+ const removed = r.status === 0;
251
+ // Best-effort launcher cleanup. Caller may pass minionsHome (so we know
252
+ // where to look); when omitted (e.g. CLI doesn't thread it through), we
253
+ // skip the file deletion — it's harmless to leave behind.
254
+ if (opts.minionsHome) {
255
+ try { fs.unlinkSync(windowsLauncherPath(opts.minionsHome)); } catch { /* ok */ }
256
+ }
257
+ return { ok: true, scheduler: 'Task Scheduler', taskName: WIN_TASK_NAME, removed };
258
+ }
259
+
260
+ function statusWindows(opts) {
261
+ const sp = opts.spawner || spawnSync;
262
+ const r = sp('schtasks.exe', ['/Query', '/TN', WIN_TASK_NAME, '/V', '/FO', 'LIST'], { encoding: 'utf8', windowsHide: true });
263
+ return {
264
+ installed: r.status === 0,
265
+ scheduler: 'Task Scheduler',
266
+ taskName: WIN_TASK_NAME,
267
+ details: (r.stdout || r.stderr || '').trim(),
268
+ };
269
+ }
270
+
271
+ // ─── macOS: launchd LaunchAgent ──────────────────────────────────────────────
272
+
273
+ function macPlistPath() {
274
+ return path.join(os.homedir(), 'Library', 'LaunchAgents', `${MAC_PLIST_LABEL}.plist`);
275
+ }
276
+
277
+ function buildMacPlist(opts) {
278
+ const stdoutPath = path.join(opts.minionsHome, 'engine', 'watchdog-launchd.log');
279
+ return `<?xml version="1.0" encoding="UTF-8"?>
280
+ <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
281
+ <plist version="1.0">
282
+ <dict>
283
+ <key>Label</key><string>${MAC_PLIST_LABEL}</string>
284
+ <key>ProgramArguments</key>
285
+ <array>
286
+ <string>${opts.nodeBin}</string>
287
+ <string>${opts.minionsBin}</string>
288
+ <string>watchdog</string>
289
+ <string>tick</string>
290
+ </array>
291
+ <key>EnvironmentVariables</key>
292
+ <dict>
293
+ <key>MINIONS_HOME</key><string>${opts.minionsHome}</string>
294
+ </dict>
295
+ <key>StartInterval</key><integer>${opts.intervalMin * 60}</integer>
296
+ <key>RunAtLoad</key><true/>
297
+ <key>StandardOutPath</key><string>${stdoutPath}</string>
298
+ <key>StandardErrorPath</key><string>${stdoutPath}</string>
299
+ </dict>
300
+ </plist>
301
+ `;
302
+ }
303
+
304
+ function installMac(opts) {
305
+ const sp = opts.spawner || spawnSync;
306
+ if (!opts.minionsHome) throw new Error('install (mac): minionsHome is required for log path and env capture');
307
+ const plistPath = macPlistPath();
308
+ fs.mkdirSync(path.dirname(plistPath), { recursive: true });
309
+ fs.writeFileSync(plistPath, buildMacPlist(opts));
310
+ // Unload first (ignore failure — agent may not be loaded) then load to
311
+ // pick up changes. Use legacy `load -w` for broad compatibility across
312
+ // 10.10+; modern `bootstrap`/`bootout` would require parsing `id -u`.
313
+ sp('launchctl', ['unload', plistPath], { stdio: 'ignore' });
314
+ const r = sp('launchctl', ['load', '-w', plistPath], { encoding: 'utf8' });
315
+ if (r.status !== 0) {
316
+ throw new Error(`launchctl load failed (exit ${r.status}): ${(r.stderr || r.stdout || '').trim()}`);
317
+ }
318
+ return { ok: true, scheduler: 'launchd', plistPath, intervalMin: opts.intervalMin };
319
+ }
320
+
321
+ function uninstallMac(opts) {
322
+ const sp = opts.spawner || spawnSync;
323
+ const plistPath = macPlistPath();
324
+ sp('launchctl', ['unload', plistPath], { stdio: 'ignore' });
325
+ let removed = false;
326
+ try { fs.unlinkSync(plistPath); removed = true; } catch { /* missing = idempotent */ }
327
+ return { ok: true, scheduler: 'launchd', plistPath, removed };
328
+ }
329
+
330
+ function statusMac(opts) {
331
+ const sp = opts.spawner || spawnSync;
332
+ const plistPath = macPlistPath();
333
+ const installed = fs.existsSync(plistPath);
334
+ const r = sp('launchctl', ['list', MAC_PLIST_LABEL], { encoding: 'utf8' });
335
+ return {
336
+ installed,
337
+ scheduler: 'launchd',
338
+ plistPath,
339
+ loaded: r.status === 0,
340
+ details: (r.stdout || r.stderr || '').trim(),
341
+ };
342
+ }
343
+
344
+ // ─── Linux: systemd --user timer ─────────────────────────────────────────────
345
+
346
+ function linuxUnitDir() {
347
+ return path.join(os.homedir(), '.config', 'systemd', 'user');
348
+ }
349
+
350
+ function buildLinuxService(opts) {
351
+ // Quote both paths so systemd's whitespace-tokenizing ExecStart parser
352
+ // doesn't split a path containing spaces (e.g. a Nix-style /nix/store
353
+ // path or a custom install under /opt/My Tools/). systemd.exec(5) supports
354
+ // double-quoted args; backslashes inside become literal so paths are safe.
355
+ const execStart = `"${opts.nodeBin}" "${opts.minionsBin}" watchdog tick`;
356
+ // Pin MINIONS_HOME for the same reason as Windows/Mac: systemd user units
357
+ // run in a stripped env that won't carry the user's interactive MINIONS_HOME
358
+ // override.
359
+ return `[Unit]
360
+ Description=Minions watchdog — probe + heal the engine/dashboard stack
361
+
362
+ [Service]
363
+ Type=oneshot
364
+ Environment="MINIONS_HOME=${opts.minionsHome}"
365
+ ExecStart=${execStart}
366
+ `;
367
+ }
368
+
369
+ function buildLinuxTimer(opts) {
370
+ return `[Unit]
371
+ Description=Minions watchdog timer (every ${opts.intervalMin} min)
372
+
373
+ [Timer]
374
+ OnBootSec=1min
375
+ OnUnitActiveSec=${opts.intervalMin}min
376
+ Unit=${LINUX_UNIT_NAME}.service
377
+
378
+ [Install]
379
+ WantedBy=timers.target
380
+ `;
381
+ }
382
+
383
+ function installLinux(opts) {
384
+ const sp = opts.spawner || spawnSync;
385
+ if (!opts.minionsHome) throw new Error('install (linux): minionsHome is required for Environment= line');
386
+ const unitDir = linuxUnitDir();
387
+ fs.mkdirSync(unitDir, { recursive: true });
388
+ fs.writeFileSync(path.join(unitDir, `${LINUX_UNIT_NAME}.service`), buildLinuxService(opts));
389
+ fs.writeFileSync(path.join(unitDir, `${LINUX_UNIT_NAME}.timer`), buildLinuxTimer(opts));
390
+ sp('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' });
391
+ const r = sp('systemctl', ['--user', 'enable', '--now', `${LINUX_UNIT_NAME}.timer`], { encoding: 'utf8' });
392
+ if (r.status !== 0) {
393
+ const userName = os.userInfo().username;
394
+ throw new Error(
395
+ `systemctl --user enable --now ${LINUX_UNIT_NAME}.timer failed (exit ${r.status}): ` +
396
+ `${(r.stderr || r.stdout || '').trim()}\n` +
397
+ `Hint: if you're not logged into a graphical session, run ` +
398
+ `'sudo loginctl enable-linger ${userName}' so the user manager survives logout, ` +
399
+ `then retry 'minions watchdog install'.`
400
+ );
401
+ }
402
+ return {
403
+ ok: true,
404
+ scheduler: 'systemd --user',
405
+ servicePath: path.join(unitDir, `${LINUX_UNIT_NAME}.service`),
406
+ timerPath: path.join(unitDir, `${LINUX_UNIT_NAME}.timer`),
407
+ intervalMin: opts.intervalMin,
408
+ lingerHint: `Run 'sudo loginctl enable-linger ${os.userInfo().username}' so the timer survives logout.`,
409
+ };
410
+ }
411
+
412
+ function uninstallLinux(opts) {
413
+ const sp = opts.spawner || spawnSync;
414
+ sp('systemctl', ['--user', 'disable', '--now', `${LINUX_UNIT_NAME}.timer`], { stdio: 'ignore' });
415
+ const unitDir = linuxUnitDir();
416
+ let removed = false;
417
+ for (const f of [`${LINUX_UNIT_NAME}.timer`, `${LINUX_UNIT_NAME}.service`]) {
418
+ try { fs.unlinkSync(path.join(unitDir, f)); removed = true; } catch { /* idempotent */ }
419
+ }
420
+ sp('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' });
421
+ return { ok: true, scheduler: 'systemd --user', removed };
422
+ }
423
+
424
+ function statusLinux(opts) {
425
+ const sp = opts.spawner || spawnSync;
426
+ const timerPath = path.join(linuxUnitDir(), `${LINUX_UNIT_NAME}.timer`);
427
+ const installed = fs.existsSync(timerPath);
428
+ const r = sp('systemctl', ['--user', 'status', `${LINUX_UNIT_NAME}.timer`, '--no-pager'], { encoding: 'utf8' });
429
+ return {
430
+ installed,
431
+ scheduler: 'systemd --user',
432
+ timerPath,
433
+ active: r.status === 0,
434
+ details: (r.stdout || r.stderr || '').trim(),
435
+ };
436
+ }
437
+
438
+ module.exports = {
439
+ tick,
440
+ install,
441
+ uninstall,
442
+ status,
443
+ isPidAlive,
444
+ // Constants + builders exported for unit tests and callers wiring CLI help.
445
+ DEFAULT_INTERVAL_MIN,
446
+ DEFAULT_DASH_PORT,
447
+ WIN_TASK_NAME,
448
+ MAC_PLIST_LABEL,
449
+ LINUX_UNIT_NAME,
450
+ WATCHDOG_LOG_NAME,
451
+ buildMacPlist,
452
+ buildLinuxService,
453
+ buildLinuxTimer,
454
+ buildWindowsLauncher,
455
+ macPlistPath,
456
+ windowsLauncherPath,
457
+ linuxUnitDir,
458
+ };
@@ -284,6 +284,13 @@ function gcDispatchWorktreeIfOrphan(opts) {
284
284
  worktreeRoot,
285
285
  log = _noopLog,
286
286
  removeWorktree = null,
287
+ // PR #3133 review — when the dispatch-end GC is the caller, the
288
+ // current dispatch's own active row legitimately claims worktreePath
289
+ // (persisted by engine.js at pending→active). Pass `excludeDispatchId`
290
+ // so shared.removeWorktree's live-guard ignores that row and lets the
291
+ // dispatch GC its own orphan worktree. Other live claimants still
292
+ // block the wipe.
293
+ excludeDispatchId = null,
287
294
  config = null,
288
295
  writeToInbox = null,
289
296
  } = opts || {};
@@ -295,9 +302,10 @@ function gcDispatchWorktreeIfOrphan(opts) {
295
302
  return { outcome: 'skip', reason: 'no-git-root', removed: false };
296
303
  }
297
304
  const _removeFn = typeof removeWorktree === 'function' ? removeWorktree : shared.removeWorktree;
305
+ const _rmOpts = excludeDispatchId ? { excludeDispatchId } : undefined;
298
306
  const resolved = (() => { try { return path.resolve(worktreePath); } catch { return worktreePath; } })();
299
307
  try {
300
- const removed = _removeFn(worktreePath, gitRoot, worktreeRoot);
308
+ const removed = _removeFn(worktreePath, gitRoot, worktreeRoot, _rmOpts);
301
309
  if (removed) {
302
310
  _markStuckSuccess(resolved, { writeToInbox });
303
311
  log('info', `worktree-gc: dispatch-end removed ${path.basename(worktreePath)}`);
package/engine.js CHANGED
@@ -1021,9 +1021,21 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
1021
1021
  statusError: e.message,
1022
1022
  },
1023
1023
  );
1024
- result.quarantined = true;
1025
- result.quarantinedPath = q.quarantinedPath;
1026
- result.backupRef = q.backupRef;
1024
+ // PR #3133 review — honor q.skipped instead of dishonestly
1025
+ // claiming quarantined=true with a null path. When
1026
+ // _quarantineDirtyWorktree skips because shared.isWorktreePathLive
1027
+ // fires on the worktree, no rename happened, no backup ref was
1028
+ // written, and the caller's downstream "quarantined to X" log /
1029
+ // error message would print "quarantined to null". Surface the
1030
+ // skip honestly so the spawn-error path (engine.js:2056-2066) can
1031
+ // render a truthful message and the dispatch stays non-retryable.
1032
+ if (q.skipped) {
1033
+ result.quarantineSkipped = true;
1034
+ } else {
1035
+ result.quarantined = true;
1036
+ result.quarantinedPath = q.quarantinedPath;
1037
+ result.backupRef = q.backupRef;
1038
+ }
1027
1039
  } catch (qErr) {
1028
1040
  result.quarantineError = qErr.message;
1029
1041
  log('error', `assertCleanSharedWorktree: quarantine after status-failed failed for ${worktreePath}: ${qErr.message}`);
@@ -1143,9 +1155,16 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
1143
1155
  dirtyFiles: result.dirtyFiles,
1144
1156
  },
1145
1157
  );
1146
- result.quarantined = true;
1147
- result.quarantinedPath = q.quarantinedPath;
1148
- result.backupRef = q.backupRef;
1158
+ // PR #3133 review — same honest-result fix as the status-failed
1159
+ // path above. If _quarantineDirtyWorktree was skipped by the
1160
+ // live-guard, do NOT set quarantined=true with a null path.
1161
+ if (q.skipped) {
1162
+ result.quarantineSkipped = true;
1163
+ } else {
1164
+ result.quarantined = true;
1165
+ result.quarantinedPath = q.quarantinedPath;
1166
+ result.backupRef = q.backupRef;
1167
+ }
1149
1168
  } catch (qErr) {
1150
1169
  result.quarantineError = qErr.message;
1151
1170
  log('error', `assertCleanSharedWorktree: quarantine failed for ${worktreePath}: ${qErr.message}`);
@@ -1214,6 +1233,14 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
1214
1233
 
1215
1234
  // Rename the worktree dir. Once this succeeds the worktree is functionally
1216
1235
  // quarantined; subsequent failures only affect ref bookkeeping.
1236
+ // W-mq5rwwss000f30a7 — never quarantine (=rename out from under) a worktree
1237
+ // that a live dispatch still claims. Quarantine breaks the agent's cwd just
1238
+ // as completely as removeWorktree would.
1239
+ if (shared.isWorktreePathLive(worktreePath)) {
1240
+ log('warn', `_quarantineDirtyWorktree: skip — live dispatch in ${worktreePath}`);
1241
+ shared._writeWorktreeSkipLiveInboxNote(worktreePath, '_quarantineDirtyWorktree');
1242
+ return { quarantinedPath: null, backupRef: null, skipped: true };
1243
+ }
1217
1244
  fs.renameSync(worktreePath, quarantinedPath);
1218
1245
 
1219
1246
  // Prune git's stale worktree metadata so the next `git worktree add` for
@@ -2047,14 +2074,14 @@ async function spawnAgent(dispatchItem, config) {
2047
2074
  const failureClassName = isDivergent ? 'WORKTREE_DIVERGENT' : 'WORKTREE_DIRTY';
2048
2075
  const reasonMsg = cleanResult.quarantined
2049
2076
  ? `${failureClassName}: reused worktree at ${worktreePath} was dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} dirty file(s)${previewFiles ? ': ' + previewFiles : ''}) — quarantined to ${cleanResult.quarantinedPath}. Next dispatch will start fresh.`
2050
- : `${failureClassName}: reused worktree at ${worktreePath} is dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} file(s)${previewFiles ? ': ' + previewFiles : ''}). Quarantine ${cleanResult.quarantineError ? 'errored: ' + cleanResult.quarantineError : 'was not attempted (' + cleanResult.reason + ').'}`;
2077
+ : `${failureClassName}: reused worktree at ${worktreePath} is dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} file(s)${previewFiles ? ': ' + previewFiles : ''}). Quarantine ${cleanResult.quarantineError ? 'errored: ' + cleanResult.quarantineError : (cleanResult.quarantineSkipped ? 'was skipped — another live dispatch claims the worktree (see notes/inbox/ engine-worktree-skip-live note).' : 'was not attempted (' + cleanResult.reason + ').')}`;
2051
2078
  log('error', reasonMsg);
2052
2079
  _cleanupPromptFiles();
2053
2080
  completeDispatch(
2054
2081
  id,
2055
2082
  DISPATCH_RESULT.ERROR,
2056
2083
  reasonMsg.slice(0, 500),
2057
- `Engine preflight refused to dispatch into a dirty/divergent reused worktree (#2996). Reason: ${cleanResult.reason}.${cleanResult.quarantined ? ` Worktree quarantined to ${cleanResult.quarantinedPath}; backup ref ${cleanResult.backupRef || '(skipped)'}. See notes/inbox/ for recovery instructions.` : ''}`,
2084
+ `Engine preflight refused to dispatch into a dirty/divergent reused worktree (#2996). Reason: ${cleanResult.reason}.${cleanResult.quarantined ? ` Worktree quarantined to ${cleanResult.quarantinedPath}; backup ref ${cleanResult.backupRef || '(skipped)'}. See notes/inbox/ for recovery instructions.` : (cleanResult.quarantineSkipped ? ' Quarantine was skipped because another live dispatch claims this worktree path; this dispatch will not auto-retry until the live claimant clears.' : '')}`,
2058
2085
  { agentRetryable: isStatusProbeFailed && cleanResult.quarantined, failureClass: failureClassValue },
2059
2086
  );
2060
2087
  cleanupTempAgent(agentId);
@@ -3743,6 +3770,15 @@ async function spawnAgent(dispatchItem, config) {
3743
3770
  const _projForReturn = project?.name || 'default';
3744
3771
  const _poolSizeReturn = worktreePool.getProjectPoolSize(_projForReturn, config);
3745
3772
  if (!_keepPidsAlive && !_managedSpawnAlive && _poolSizeReturn > 0) {
3773
+ // W-mq5rwwss000f30a7 — defensive live-worktree check before the
3774
+ // destructive `git reset --hard HEAD → git clean -fd → checkout
3775
+ // --detach` chain. We pass excludeDispatchId so the active row for
3776
+ // THIS dispatch (still in dispatch.active until completeDispatch
3777
+ // fires below) is ignored. Any OTHER live dispatch claiming the
3778
+ // same path is a sign of a concurrency bug and must not be wiped.
3779
+ if (shared.isWorktreePathLive(worktreePath, { excludeDispatchId: id })) {
3780
+ log('warn', `worktree-pool: skip return — another live dispatch claims ${worktreePath}`);
3781
+ } else {
3746
3782
  try {
3747
3783
  const _mainRefRet = sanitizeBranch(shared.resolveMainBranch(rootDir, project?.mainBranch));
3748
3784
  await shared.shellSafeGit(['reset', '--hard', 'HEAD'], { ..._gitOpts, cwd: worktreePath, timeout: 30000 });
@@ -3771,6 +3807,7 @@ async function spawnAgent(dispatchItem, config) {
3771
3807
  // dispatch-end GC below will pick it up via the isPoolMember
3772
3808
  // check (which now correctly returns false).
3773
3809
  }
3810
+ }
3774
3811
  } else if (_keepPidsAlive || _managedSpawnAlive) {
3775
3812
  // Skip the pool — the worktree is in use by left-running processes
3776
3813
  // (keep_processes PIDs or managed-spawn services). Make sure no
@@ -3803,6 +3840,13 @@ async function spawnAgent(dispatchItem, config) {
3803
3840
  worktreeRoot: _wtRoot,
3804
3841
  agentId,
3805
3842
  managedSpawnSpawnedCount: Array.isArray(managedSpawnSpawned) ? managedSpawnSpawned.length : 0,
3843
+ // W-mq5rwwss000f30a7 / PR #3133 review — this dispatch's own
3844
+ // active row still claims worktreePath (persisted at pending→active,
3845
+ // line ~3998). Without excludeDispatchId, shared.removeWorktree's
3846
+ // live-guard would skip and silently neuter the GC for the
3847
+ // default worktreePoolSize:0 config. Pool-return uses the same
3848
+ // plumbing; this is the matching wiring on the orphan-GC side.
3849
+ excludeDispatchId: id,
3806
3850
  log,
3807
3851
  });
3808
3852
  if (_gcResult.outcome === 'gc') {
@@ -3973,6 +4017,11 @@ async function spawnAgent(dispatchItem, config) {
3973
4017
  // route output parsing through the right adapter. Also surfaces the choice
3974
4018
  // in dispatch.json for debugging multi-runtime fleets.
3975
4019
  item.runtimeName = runtimeName;
4020
+ // W-mq5rwwss000f30a7 — persist the worktree path so the live-worktree
4021
+ // guard (shared.isWorktreePathLive) can correlate destructive callers
4022
+ // (removeWorktree, cleanup orphan-dir sweep, pool-return, quarantine)
4023
+ // back to this active dispatch and skip the wipe.
4024
+ if (worktreePath) item.worktreePath = worktreePath;
3976
4025
  delete item.skipReason;
3977
4026
  delete item._agentBusySince;
3978
4027
  if (!dispatch.active.some(d => d.id === id)) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@yemi33/minions",
3
- "version": "0.1.2144",
3
+ "version": "0.1.2146",
4
4
  "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
5
5
  "bin": {
6
6
  "minions": "bin/minions.js"