@yemi33/minions 0.1.2144 → 0.1.2146
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/bin/minions.js +85 -0
- package/bin/minions.js.rej +16 -0
- package/dashboard/js/refresh.js +14 -0
- package/dashboard/js/render-pinned.js +119 -3
- package/dashboard/js/utils.js +20 -0
- package/dashboard/layout.html +1 -0
- package/dashboard/slim/body.html +113 -1
- package/dashboard/slim/body.html.rej +11 -0
- package/dashboard/slim/js/command-send.js.rej +12 -0
- package/dashboard/slim/js/helpers.js +9 -0
- package/dashboard/slim/js/history.js +153 -88
- package/dashboard/slim/js/history.js.rej +26 -0
- package/dashboard/slim/js/modals-tiles.js +8 -2
- package/dashboard/slim/js/pinned.js +182 -0
- package/dashboard/slim/js/settings.js +126 -6
- package/dashboard/slim/js/status.js +9 -6
- package/dashboard/slim/layout.html +1 -0
- package/dashboard/slim/styles.css +77 -2
- package/dashboard/slim/styles.css.rej +124 -0
- package/dashboard/styles.css +19 -0
- package/dashboard-build.js +9 -2
- package/dashboard.js +44 -1
- package/docs/README.md.rej +9 -0
- package/docs/auto-discovery.md +2 -2
- package/docs/constellation-style-telemetry.md +161 -0
- package/docs/engine-restart.md +1 -1
- package/docs/kb-sweep.md +2 -2
- package/docs/managed-spawn.md +1 -1
- package/docs/watches.md +11 -11
- package/engine/cleanup.js +9 -0
- package/engine/cli.js +57 -12
- package/engine/features.js +11 -0
- package/engine/shared.js +290 -37
- package/engine/watchdog.js +458 -0
- package/engine/worktree-gc.js +9 -1
- package/engine.js +57 -8
- package/package.json +1 -1
|
@@ -0,0 +1,458 @@
|
|
|
1
|
+
// engine/watchdog.js — out-of-process recovery scheduled by the OS.
|
|
2
|
+
//
|
|
3
|
+
// PROBLEM (cluster-kill):
|
|
4
|
+
// The in-process supervisor (engine/supervisor.js) respawns engine/dashboard
|
|
5
|
+
// when one of them dies in isolation, but it can't respawn ITSELF. When
|
|
6
|
+
// something kills all three at once — Windows job-object teardown when a
|
|
7
|
+
// parent terminal closes (RDP disconnect, Windows Terminal auto-update,
|
|
8
|
+
// manual close), Linux OOM-killer, systemd cgroup memory limits, manual
|
|
9
|
+
// `pkill -f node`, system reboot — recovery requires an external agent.
|
|
10
|
+
//
|
|
11
|
+
// SOLUTION:
|
|
12
|
+
// Register `minions watchdog tick` with the OS scheduler (Task Scheduler /
|
|
13
|
+
// launchd / systemd --user) to run every N minutes. Each tick probes the
|
|
14
|
+
// stack, logs the verdict, and invokes `minions start`/`restart` if dead.
|
|
15
|
+
// The scheduler itself lives outside any user terminal's job/session tree,
|
|
16
|
+
// so it survives every cluster-kill scenario that takes minions down.
|
|
17
|
+
//
|
|
18
|
+
// CONTRACT:
|
|
19
|
+
// tick(opts) → always resolves; never throws to the caller.
|
|
20
|
+
// The scheduler must never see a non-zero exit,
|
|
21
|
+
// or it may start escalating (eventvwr, syslog,
|
|
22
|
+
// email-on-failure) for the very recovery we're
|
|
23
|
+
// trying to make boring.
|
|
24
|
+
// install(opts) → throws on platform-tool failure (so the user
|
|
25
|
+
// sees it; install is interactive).
|
|
26
|
+
// uninstall() → idempotent; removing a non-existent
|
|
27
|
+
// registration is success, not error.
|
|
28
|
+
// status() → returns { installed, scheduler, … details }.
|
|
29
|
+
//
|
|
30
|
+
// Cross-platform note: install/uninstall/status branch on process.platform.
|
|
31
|
+
// Each platform's helper is self-contained so unit tests can stub spawnSync.
|
|
32
|
+
|
|
33
|
+
const fs = require('fs');
|
|
34
|
+
const path = require('path');
|
|
35
|
+
const os = require('os');
|
|
36
|
+
const { spawn, spawnSync } = require('child_process');
|
|
37
|
+
|
|
38
|
+
const DEFAULT_INTERVAL_MIN = 5;
|
|
39
|
+
const DEFAULT_DASH_PORT = 7331;
|
|
40
|
+
const WIN_TASK_NAME = 'MinionsWatchdog';
|
|
41
|
+
const MAC_PLIST_LABEL = 'io.minions.watchdog';
|
|
42
|
+
const LINUX_UNIT_NAME = 'minions-watchdog';
|
|
43
|
+
const TICK_SPAWN_TIMEOUT_MS = 120000;
|
|
44
|
+
const WATCHDOG_LOG_NAME = 'watchdog-stdio.log';
|
|
45
|
+
// Cap the watchdog log so a misbehaving scheduler can't fill the disk
|
|
46
|
+
// over months. Truncates to half-size on overflow (oldest lines dropped).
|
|
47
|
+
const WATCHDOG_LOG_MAX_BYTES = 2 * 1024 * 1024;
|
|
48
|
+
|
|
49
|
+
function logLine(minionsHome, msg) {
|
|
50
|
+
try {
|
|
51
|
+
const dir = path.join(minionsHome, 'engine');
|
|
52
|
+
fs.mkdirSync(dir, { recursive: true });
|
|
53
|
+
const logPath = path.join(dir, WATCHDOG_LOG_NAME);
|
|
54
|
+
try {
|
|
55
|
+
const stat = fs.statSync(logPath);
|
|
56
|
+
if (stat.size > WATCHDOG_LOG_MAX_BYTES) {
|
|
57
|
+
const buf = fs.readFileSync(logPath);
|
|
58
|
+
const tail = buf.subarray(Math.floor(buf.length / 2));
|
|
59
|
+
const firstNewline = tail.indexOf(0x0a);
|
|
60
|
+
const trimmed = firstNewline >= 0 ? tail.subarray(firstNewline + 1) : tail;
|
|
61
|
+
fs.writeFileSync(logPath, trimmed);
|
|
62
|
+
}
|
|
63
|
+
} catch { /* file may not exist yet — that's fine */ }
|
|
64
|
+
fs.appendFileSync(logPath, `[${new Date().toISOString()}] ${msg}\n`);
|
|
65
|
+
} catch { /* logging failure must never escalate to the scheduler */ }
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
function isPidAlive(pid) {
|
|
69
|
+
if (!pid || !Number.isInteger(pid) || pid <= 0) return false;
|
|
70
|
+
try { process.kill(pid, 0); return true; } catch { return false; }
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Probe engine + dashboard and recover if needed. Always resolves (never
|
|
75
|
+
* throws). Returns { healthy, action, exitCode? } describing what was done.
|
|
76
|
+
*
|
|
77
|
+
* Recovery policy:
|
|
78
|
+
* - stop-intent flag set → STAND DOWN (user wanted minions stopped;
|
|
79
|
+
* respawning would undo `minions stop`)
|
|
80
|
+
* - both alive → no-op
|
|
81
|
+
* - both dead → `minions start` (idempotent fast path)
|
|
82
|
+
* - one alive one dead → `minions restart` (only restart can
|
|
83
|
+
* reconcile partial state — start refuses
|
|
84
|
+
* by design)
|
|
85
|
+
*/
|
|
86
|
+
async function tick(opts) {
|
|
87
|
+
opts = opts || {};
|
|
88
|
+
const minionsHome = opts.minionsHome;
|
|
89
|
+
const minionsBin = opts.minionsBin;
|
|
90
|
+
const dashPort = opts.dashPort || DEFAULT_DASH_PORT;
|
|
91
|
+
const readEnginePid = opts.readEnginePid;
|
|
92
|
+
const isPortListening = opts.isPortListening;
|
|
93
|
+
const isStopIntentSet = opts.isStopIntentSet;
|
|
94
|
+
const spawner = opts.spawner || spawn;
|
|
95
|
+
const now = opts.now || (() => new Date());
|
|
96
|
+
|
|
97
|
+
if (!minionsHome || !minionsBin || !readEnginePid || !isPortListening) {
|
|
98
|
+
// Programmer error — caller forgot a dep. Try to log when minionsHome is
|
|
99
|
+
// available (logLine swallows errors of its own), then resolve with a
|
|
100
|
+
// marker. The scheduler must never see a non-zero exit.
|
|
101
|
+
if (minionsHome) logLine(minionsHome, 'missing-deps → noop (caller did not provide all required helpers)');
|
|
102
|
+
return { healthy: false, action: 'noop', error: 'missing-deps' };
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// Respect `minions stop` / `minions uninstall` / mid-restart windows.
|
|
106
|
+
// On forks that ship an in-process supervisor (yemi33/minions), this flag
|
|
107
|
+
// is the same one the supervisor uses to suppress engine respawns; the
|
|
108
|
+
// watchdog MUST honor it too, or it would brain-deadly restart minions
|
|
109
|
+
// every N minutes after the user explicitly stopped it.
|
|
110
|
+
// On forks that DON'T ship a supervisor + stop-intent producer (e.g.
|
|
111
|
+
// opg-microsoft/minions), `isStopIntentSet` is intentionally absent — we
|
|
112
|
+
// treat that as "no stop-intent system, proceed with recovery" (fail-open).
|
|
113
|
+
let stopWanted = false;
|
|
114
|
+
if (typeof isStopIntentSet === 'function') {
|
|
115
|
+
try { stopWanted = !!isStopIntentSet(); } catch { stopWanted = false; }
|
|
116
|
+
}
|
|
117
|
+
if (stopWanted) {
|
|
118
|
+
logLine(minionsHome, `stop-intent set → standing down (no probe, no recovery)`);
|
|
119
|
+
return { healthy: false, action: 'stand-down' };
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
let enginePid = null;
|
|
123
|
+
try { enginePid = readEnginePid(minionsHome); } catch { enginePid = null; }
|
|
124
|
+
const engineAlive = isPidAlive(enginePid);
|
|
125
|
+
let dashUp = false;
|
|
126
|
+
try { dashUp = !!isPortListening(dashPort); } catch { dashUp = false; }
|
|
127
|
+
|
|
128
|
+
if (engineAlive && dashUp) {
|
|
129
|
+
logLine(minionsHome, `ok engine=${enginePid} port=${dashPort} ts=${now().toISOString()}`);
|
|
130
|
+
return { healthy: true, action: 'none' };
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const partial = engineAlive || dashUp;
|
|
134
|
+
const action = partial ? 'restart' : 'start';
|
|
135
|
+
logLine(
|
|
136
|
+
minionsHome,
|
|
137
|
+
`unhealthy engine=${enginePid || '-'}(${engineAlive ? 'alive' : 'dead'}) ` +
|
|
138
|
+
`port=${dashPort}(${dashUp ? 'up' : 'down'}) → invoking minions ${action}`
|
|
139
|
+
);
|
|
140
|
+
|
|
141
|
+
try {
|
|
142
|
+
// Detach the child so its lifetime is independent of this tick process.
|
|
143
|
+
// The OS scheduler closes our stdio shortly after exit; we don't want
|
|
144
|
+
// that closure to cascade into the new daemon stack.
|
|
145
|
+
const child = spawner(process.execPath, [minionsBin, action], {
|
|
146
|
+
detached: true,
|
|
147
|
+
stdio: 'ignore',
|
|
148
|
+
windowsHide: true,
|
|
149
|
+
});
|
|
150
|
+
if (child && typeof child.unref === 'function') child.unref();
|
|
151
|
+
logLine(minionsHome, `spawned minions ${action} pid=${child && child.pid} (detached)`);
|
|
152
|
+
return { healthy: false, action, spawnedPid: child && child.pid };
|
|
153
|
+
} catch (err) {
|
|
154
|
+
logLine(minionsHome, `FAILED to spawn minions ${action}: ${err && err.message || err}`);
|
|
155
|
+
return { healthy: false, action, error: String(err && err.message || err) };
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
// Documented ceiling for any future synchronous variant of tick. Not used
|
|
159
|
+
// by the detached spawn path, but kept reachable so callers can opt in.
|
|
160
|
+
tick.TICK_SPAWN_TIMEOUT_MS = TICK_SPAWN_TIMEOUT_MS;
|
|
161
|
+
|
|
162
|
+
// ─── Install / uninstall / status ────────────────────────────────────────────
|
|
163
|
+
|
|
164
|
+
function install(opts) {
|
|
165
|
+
opts = opts || {};
|
|
166
|
+
const plat = opts.platform || process.platform;
|
|
167
|
+
const minionsBin = opts.minionsBin;
|
|
168
|
+
const intervalMin = Number.isFinite(opts.intervalMin) && opts.intervalMin > 0
|
|
169
|
+
? Math.floor(opts.intervalMin)
|
|
170
|
+
: DEFAULT_INTERVAL_MIN;
|
|
171
|
+
const nodeBin = opts.nodeBin || process.execPath;
|
|
172
|
+
if (!minionsBin) throw new Error('install: minionsBin is required');
|
|
173
|
+
|
|
174
|
+
if (plat === 'win32') return installWindows({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
|
|
175
|
+
if (plat === 'darwin') return installMac({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
|
|
176
|
+
if (plat === 'linux') return installLinux({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
|
|
177
|
+
throw new Error(`watchdog install: unsupported platform "${plat}"`);
|
|
178
|
+
}
|
|
179
|
+
|
|
180
|
+
function uninstall(opts) {
|
|
181
|
+
opts = opts || {};
|
|
182
|
+
const plat = opts.platform || process.platform;
|
|
183
|
+
if (plat === 'win32') return uninstallWindows({ minionsHome: opts.minionsHome, spawner: opts.spawner });
|
|
184
|
+
if (plat === 'darwin') return uninstallMac({ spawner: opts.spawner });
|
|
185
|
+
if (plat === 'linux') return uninstallLinux({ spawner: opts.spawner });
|
|
186
|
+
throw new Error(`watchdog uninstall: unsupported platform "${plat}"`);
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function status(opts) {
|
|
190
|
+
opts = opts || {};
|
|
191
|
+
const plat = opts.platform || process.platform;
|
|
192
|
+
if (plat === 'win32') return statusWindows({ spawner: opts.spawner });
|
|
193
|
+
if (plat === 'darwin') return statusMac({ spawner: opts.spawner });
|
|
194
|
+
if (plat === 'linux') return statusLinux({ spawner: opts.spawner });
|
|
195
|
+
throw new Error(`watchdog status: unsupported platform "${plat}"`);
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
// ─── Windows: Task Scheduler ─────────────────────────────────────────────────
|
|
199
|
+
|
|
200
|
+
// On Windows we generate a launcher .cmd in MINIONS_HOME/engine/ that sets
|
|
201
|
+
// MINIONS_HOME before invoking node, then point schtasks at that file. This
|
|
202
|
+
// avoids two failure modes that the obvious schtasks /TR "node bin tick"
|
|
203
|
+
// form has:
|
|
204
|
+
// (a) The task runs without MINIONS_HOME in env (Task Scheduler doesn't
|
|
205
|
+
// inherit the install-time user session env), so a user with a
|
|
206
|
+
// non-default MINIONS_HOME would have the watchdog probe and "recover"
|
|
207
|
+
// the WRONG stack (~/.minions instead of their real install).
|
|
208
|
+
// (b) /TR has hard length + quoting limits for nested arguments. A
|
|
209
|
+
// launcher script keeps schtasks's /TR simple and gives the user a
|
|
210
|
+
// readable, auditable file to inspect on disk.
|
|
211
|
+
function buildWindowsLauncher(opts) {
|
|
212
|
+
return `@echo off\r
|
|
213
|
+
REM Minions watchdog launcher — auto-generated by 'minions watchdog install'.\r
|
|
214
|
+
REM Re-run 'minions watchdog install' to regenerate after moving MINIONS_HOME.\r
|
|
215
|
+
set "MINIONS_HOME=${opts.minionsHome}"\r
|
|
216
|
+
"${opts.nodeBin}" "${opts.minionsBin}" watchdog tick\r
|
|
217
|
+
`;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
function windowsLauncherPath(minionsHome) {
|
|
221
|
+
return path.join(minionsHome, 'engine', 'watchdog-launcher.cmd');
|
|
222
|
+
}
|
|
223
|
+
|
|
224
|
+
function installWindows(opts) {
|
|
225
|
+
const sp = opts.spawner || spawnSync;
|
|
226
|
+
if (!opts.minionsHome) throw new Error('install (win32): minionsHome is required for launcher path');
|
|
227
|
+
const launcherPath = windowsLauncherPath(opts.minionsHome);
|
|
228
|
+
fs.mkdirSync(path.dirname(launcherPath), { recursive: true });
|
|
229
|
+
fs.writeFileSync(launcherPath, buildWindowsLauncher(opts));
|
|
230
|
+
// /TR points at the launcher. Quote it so spaces in MINIONS_HOME survive.
|
|
231
|
+
const tr = `"${launcherPath}"`;
|
|
232
|
+
const args = [
|
|
233
|
+
'/Create', '/F',
|
|
234
|
+
'/SC', 'MINUTE', '/MO', String(opts.intervalMin),
|
|
235
|
+
'/TN', WIN_TASK_NAME,
|
|
236
|
+
'/TR', tr,
|
|
237
|
+
'/RL', 'LIMITED',
|
|
238
|
+
];
|
|
239
|
+
const r = sp('schtasks.exe', args, { encoding: 'utf8', windowsHide: true });
|
|
240
|
+
if (r.status !== 0) {
|
|
241
|
+
throw new Error(`schtasks /Create failed (exit ${r.status}): ${(r.stderr || r.stdout || '').trim()}`);
|
|
242
|
+
}
|
|
243
|
+
return { ok: true, scheduler: 'Task Scheduler', taskName: WIN_TASK_NAME, intervalMin: opts.intervalMin, launcherPath };
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function uninstallWindows(opts) {
|
|
247
|
+
const sp = opts.spawner || spawnSync;
|
|
248
|
+
const r = sp('schtasks.exe', ['/Delete', '/F', '/TN', WIN_TASK_NAME], { encoding: 'utf8', windowsHide: true });
|
|
249
|
+
// Exit 1 from schtasks usually means "task not found" — idempotent success.
|
|
250
|
+
const removed = r.status === 0;
|
|
251
|
+
// Best-effort launcher cleanup. Caller may pass minionsHome (so we know
|
|
252
|
+
// where to look); when omitted (e.g. CLI doesn't thread it through), we
|
|
253
|
+
// skip the file deletion — it's harmless to leave behind.
|
|
254
|
+
if (opts.minionsHome) {
|
|
255
|
+
try { fs.unlinkSync(windowsLauncherPath(opts.minionsHome)); } catch { /* ok */ }
|
|
256
|
+
}
|
|
257
|
+
return { ok: true, scheduler: 'Task Scheduler', taskName: WIN_TASK_NAME, removed };
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
function statusWindows(opts) {
|
|
261
|
+
const sp = opts.spawner || spawnSync;
|
|
262
|
+
const r = sp('schtasks.exe', ['/Query', '/TN', WIN_TASK_NAME, '/V', '/FO', 'LIST'], { encoding: 'utf8', windowsHide: true });
|
|
263
|
+
return {
|
|
264
|
+
installed: r.status === 0,
|
|
265
|
+
scheduler: 'Task Scheduler',
|
|
266
|
+
taskName: WIN_TASK_NAME,
|
|
267
|
+
details: (r.stdout || r.stderr || '').trim(),
|
|
268
|
+
};
|
|
269
|
+
}
|
|
270
|
+
|
|
271
|
+
// ─── macOS: launchd LaunchAgent ──────────────────────────────────────────────
|
|
272
|
+
|
|
273
|
+
function macPlistPath() {
|
|
274
|
+
return path.join(os.homedir(), 'Library', 'LaunchAgents', `${MAC_PLIST_LABEL}.plist`);
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
function buildMacPlist(opts) {
|
|
278
|
+
const stdoutPath = path.join(opts.minionsHome, 'engine', 'watchdog-launchd.log');
|
|
279
|
+
return `<?xml version="1.0" encoding="UTF-8"?>
|
|
280
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
|
281
|
+
<plist version="1.0">
|
|
282
|
+
<dict>
|
|
283
|
+
<key>Label</key><string>${MAC_PLIST_LABEL}</string>
|
|
284
|
+
<key>ProgramArguments</key>
|
|
285
|
+
<array>
|
|
286
|
+
<string>${opts.nodeBin}</string>
|
|
287
|
+
<string>${opts.minionsBin}</string>
|
|
288
|
+
<string>watchdog</string>
|
|
289
|
+
<string>tick</string>
|
|
290
|
+
</array>
|
|
291
|
+
<key>EnvironmentVariables</key>
|
|
292
|
+
<dict>
|
|
293
|
+
<key>MINIONS_HOME</key><string>${opts.minionsHome}</string>
|
|
294
|
+
</dict>
|
|
295
|
+
<key>StartInterval</key><integer>${opts.intervalMin * 60}</integer>
|
|
296
|
+
<key>RunAtLoad</key><true/>
|
|
297
|
+
<key>StandardOutPath</key><string>${stdoutPath}</string>
|
|
298
|
+
<key>StandardErrorPath</key><string>${stdoutPath}</string>
|
|
299
|
+
</dict>
|
|
300
|
+
</plist>
|
|
301
|
+
`;
|
|
302
|
+
}
|
|
303
|
+
|
|
304
|
+
function installMac(opts) {
|
|
305
|
+
const sp = opts.spawner || spawnSync;
|
|
306
|
+
if (!opts.minionsHome) throw new Error('install (mac): minionsHome is required for log path and env capture');
|
|
307
|
+
const plistPath = macPlistPath();
|
|
308
|
+
fs.mkdirSync(path.dirname(plistPath), { recursive: true });
|
|
309
|
+
fs.writeFileSync(plistPath, buildMacPlist(opts));
|
|
310
|
+
// Unload first (ignore failure — agent may not be loaded) then load to
|
|
311
|
+
// pick up changes. Use legacy `load -w` for broad compatibility across
|
|
312
|
+
// 10.10+; modern `bootstrap`/`bootout` would require parsing `id -u`.
|
|
313
|
+
sp('launchctl', ['unload', plistPath], { stdio: 'ignore' });
|
|
314
|
+
const r = sp('launchctl', ['load', '-w', plistPath], { encoding: 'utf8' });
|
|
315
|
+
if (r.status !== 0) {
|
|
316
|
+
throw new Error(`launchctl load failed (exit ${r.status}): ${(r.stderr || r.stdout || '').trim()}`);
|
|
317
|
+
}
|
|
318
|
+
return { ok: true, scheduler: 'launchd', plistPath, intervalMin: opts.intervalMin };
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
function uninstallMac(opts) {
|
|
322
|
+
const sp = opts.spawner || spawnSync;
|
|
323
|
+
const plistPath = macPlistPath();
|
|
324
|
+
sp('launchctl', ['unload', plistPath], { stdio: 'ignore' });
|
|
325
|
+
let removed = false;
|
|
326
|
+
try { fs.unlinkSync(plistPath); removed = true; } catch { /* missing = idempotent */ }
|
|
327
|
+
return { ok: true, scheduler: 'launchd', plistPath, removed };
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
function statusMac(opts) {
|
|
331
|
+
const sp = opts.spawner || spawnSync;
|
|
332
|
+
const plistPath = macPlistPath();
|
|
333
|
+
const installed = fs.existsSync(plistPath);
|
|
334
|
+
const r = sp('launchctl', ['list', MAC_PLIST_LABEL], { encoding: 'utf8' });
|
|
335
|
+
return {
|
|
336
|
+
installed,
|
|
337
|
+
scheduler: 'launchd',
|
|
338
|
+
plistPath,
|
|
339
|
+
loaded: r.status === 0,
|
|
340
|
+
details: (r.stdout || r.stderr || '').trim(),
|
|
341
|
+
};
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// ─── Linux: systemd --user timer ─────────────────────────────────────────────
|
|
345
|
+
|
|
346
|
+
function linuxUnitDir() {
|
|
347
|
+
return path.join(os.homedir(), '.config', 'systemd', 'user');
|
|
348
|
+
}
|
|
349
|
+
|
|
350
|
+
function buildLinuxService(opts) {
|
|
351
|
+
// Quote both paths so systemd's whitespace-tokenizing ExecStart parser
|
|
352
|
+
// doesn't split a path containing spaces (e.g. a Nix-style /nix/store
|
|
353
|
+
// path or a custom install under /opt/My Tools/). systemd.exec(5) supports
|
|
354
|
+
// double-quoted args; backslashes inside become literal so paths are safe.
|
|
355
|
+
const execStart = `"${opts.nodeBin}" "${opts.minionsBin}" watchdog tick`;
|
|
356
|
+
// Pin MINIONS_HOME for the same reason as Windows/Mac: systemd user units
|
|
357
|
+
// run in a stripped env that won't carry the user's interactive MINIONS_HOME
|
|
358
|
+
// override.
|
|
359
|
+
return `[Unit]
|
|
360
|
+
Description=Minions watchdog — probe + heal the engine/dashboard stack
|
|
361
|
+
|
|
362
|
+
[Service]
|
|
363
|
+
Type=oneshot
|
|
364
|
+
Environment="MINIONS_HOME=${opts.minionsHome}"
|
|
365
|
+
ExecStart=${execStart}
|
|
366
|
+
`;
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
function buildLinuxTimer(opts) {
|
|
370
|
+
return `[Unit]
|
|
371
|
+
Description=Minions watchdog timer (every ${opts.intervalMin} min)
|
|
372
|
+
|
|
373
|
+
[Timer]
|
|
374
|
+
OnBootSec=1min
|
|
375
|
+
OnUnitActiveSec=${opts.intervalMin}min
|
|
376
|
+
Unit=${LINUX_UNIT_NAME}.service
|
|
377
|
+
|
|
378
|
+
[Install]
|
|
379
|
+
WantedBy=timers.target
|
|
380
|
+
`;
|
|
381
|
+
}
|
|
382
|
+
|
|
383
|
+
function installLinux(opts) {
|
|
384
|
+
const sp = opts.spawner || spawnSync;
|
|
385
|
+
if (!opts.minionsHome) throw new Error('install (linux): minionsHome is required for Environment= line');
|
|
386
|
+
const unitDir = linuxUnitDir();
|
|
387
|
+
fs.mkdirSync(unitDir, { recursive: true });
|
|
388
|
+
fs.writeFileSync(path.join(unitDir, `${LINUX_UNIT_NAME}.service`), buildLinuxService(opts));
|
|
389
|
+
fs.writeFileSync(path.join(unitDir, `${LINUX_UNIT_NAME}.timer`), buildLinuxTimer(opts));
|
|
390
|
+
sp('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' });
|
|
391
|
+
const r = sp('systemctl', ['--user', 'enable', '--now', `${LINUX_UNIT_NAME}.timer`], { encoding: 'utf8' });
|
|
392
|
+
if (r.status !== 0) {
|
|
393
|
+
const userName = os.userInfo().username;
|
|
394
|
+
throw new Error(
|
|
395
|
+
`systemctl --user enable --now ${LINUX_UNIT_NAME}.timer failed (exit ${r.status}): ` +
|
|
396
|
+
`${(r.stderr || r.stdout || '').trim()}\n` +
|
|
397
|
+
`Hint: if you're not logged into a graphical session, run ` +
|
|
398
|
+
`'sudo loginctl enable-linger ${userName}' so the user manager survives logout, ` +
|
|
399
|
+
`then retry 'minions watchdog install'.`
|
|
400
|
+
);
|
|
401
|
+
}
|
|
402
|
+
return {
|
|
403
|
+
ok: true,
|
|
404
|
+
scheduler: 'systemd --user',
|
|
405
|
+
servicePath: path.join(unitDir, `${LINUX_UNIT_NAME}.service`),
|
|
406
|
+
timerPath: path.join(unitDir, `${LINUX_UNIT_NAME}.timer`),
|
|
407
|
+
intervalMin: opts.intervalMin,
|
|
408
|
+
lingerHint: `Run 'sudo loginctl enable-linger ${os.userInfo().username}' so the timer survives logout.`,
|
|
409
|
+
};
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
function uninstallLinux(opts) {
|
|
413
|
+
const sp = opts.spawner || spawnSync;
|
|
414
|
+
sp('systemctl', ['--user', 'disable', '--now', `${LINUX_UNIT_NAME}.timer`], { stdio: 'ignore' });
|
|
415
|
+
const unitDir = linuxUnitDir();
|
|
416
|
+
let removed = false;
|
|
417
|
+
for (const f of [`${LINUX_UNIT_NAME}.timer`, `${LINUX_UNIT_NAME}.service`]) {
|
|
418
|
+
try { fs.unlinkSync(path.join(unitDir, f)); removed = true; } catch { /* idempotent */ }
|
|
419
|
+
}
|
|
420
|
+
sp('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' });
|
|
421
|
+
return { ok: true, scheduler: 'systemd --user', removed };
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
function statusLinux(opts) {
|
|
425
|
+
const sp = opts.spawner || spawnSync;
|
|
426
|
+
const timerPath = path.join(linuxUnitDir(), `${LINUX_UNIT_NAME}.timer`);
|
|
427
|
+
const installed = fs.existsSync(timerPath);
|
|
428
|
+
const r = sp('systemctl', ['--user', 'status', `${LINUX_UNIT_NAME}.timer`, '--no-pager'], { encoding: 'utf8' });
|
|
429
|
+
return {
|
|
430
|
+
installed,
|
|
431
|
+
scheduler: 'systemd --user',
|
|
432
|
+
timerPath,
|
|
433
|
+
active: r.status === 0,
|
|
434
|
+
details: (r.stdout || r.stderr || '').trim(),
|
|
435
|
+
};
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
module.exports = {
|
|
439
|
+
tick,
|
|
440
|
+
install,
|
|
441
|
+
uninstall,
|
|
442
|
+
status,
|
|
443
|
+
isPidAlive,
|
|
444
|
+
// Constants + builders exported for unit tests and callers wiring CLI help.
|
|
445
|
+
DEFAULT_INTERVAL_MIN,
|
|
446
|
+
DEFAULT_DASH_PORT,
|
|
447
|
+
WIN_TASK_NAME,
|
|
448
|
+
MAC_PLIST_LABEL,
|
|
449
|
+
LINUX_UNIT_NAME,
|
|
450
|
+
WATCHDOG_LOG_NAME,
|
|
451
|
+
buildMacPlist,
|
|
452
|
+
buildLinuxService,
|
|
453
|
+
buildLinuxTimer,
|
|
454
|
+
buildWindowsLauncher,
|
|
455
|
+
macPlistPath,
|
|
456
|
+
windowsLauncherPath,
|
|
457
|
+
linuxUnitDir,
|
|
458
|
+
};
|
package/engine/worktree-gc.js
CHANGED
|
@@ -284,6 +284,13 @@ function gcDispatchWorktreeIfOrphan(opts) {
|
|
|
284
284
|
worktreeRoot,
|
|
285
285
|
log = _noopLog,
|
|
286
286
|
removeWorktree = null,
|
|
287
|
+
// PR #3133 review — when the dispatch-end GC is the caller, the
|
|
288
|
+
// current dispatch's own active row legitimately claims worktreePath
|
|
289
|
+
// (persisted by engine.js at pending→active). Pass `excludeDispatchId`
|
|
290
|
+
// so shared.removeWorktree's live-guard ignores that row and lets the
|
|
291
|
+
// dispatch GC its own orphan worktree. Other live claimants still
|
|
292
|
+
// block the wipe.
|
|
293
|
+
excludeDispatchId = null,
|
|
287
294
|
config = null,
|
|
288
295
|
writeToInbox = null,
|
|
289
296
|
} = opts || {};
|
|
@@ -295,9 +302,10 @@ function gcDispatchWorktreeIfOrphan(opts) {
|
|
|
295
302
|
return { outcome: 'skip', reason: 'no-git-root', removed: false };
|
|
296
303
|
}
|
|
297
304
|
const _removeFn = typeof removeWorktree === 'function' ? removeWorktree : shared.removeWorktree;
|
|
305
|
+
const _rmOpts = excludeDispatchId ? { excludeDispatchId } : undefined;
|
|
298
306
|
const resolved = (() => { try { return path.resolve(worktreePath); } catch { return worktreePath; } })();
|
|
299
307
|
try {
|
|
300
|
-
const removed = _removeFn(worktreePath, gitRoot, worktreeRoot);
|
|
308
|
+
const removed = _removeFn(worktreePath, gitRoot, worktreeRoot, _rmOpts);
|
|
301
309
|
if (removed) {
|
|
302
310
|
_markStuckSuccess(resolved, { writeToInbox });
|
|
303
311
|
log('info', `worktree-gc: dispatch-end removed ${path.basename(worktreePath)}`);
|
package/engine.js
CHANGED
|
@@ -1021,9 +1021,21 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
|
|
|
1021
1021
|
statusError: e.message,
|
|
1022
1022
|
},
|
|
1023
1023
|
);
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1024
|
+
// PR #3133 review — honor q.skipped instead of dishonestly
|
|
1025
|
+
// claiming quarantined=true with a null path. When
|
|
1026
|
+
// _quarantineDirtyWorktree skips because shared.isWorktreePathLive
|
|
1027
|
+
// fires on the worktree, no rename happened, no backup ref was
|
|
1028
|
+
// written, and the caller's downstream "quarantined to X" log /
|
|
1029
|
+
// error message would print "quarantined to null". Surface the
|
|
1030
|
+
// skip honestly so the spawn-error path (engine.js:2056-2066) can
|
|
1031
|
+
// render a truthful message and the dispatch stays non-retryable.
|
|
1032
|
+
if (q.skipped) {
|
|
1033
|
+
result.quarantineSkipped = true;
|
|
1034
|
+
} else {
|
|
1035
|
+
result.quarantined = true;
|
|
1036
|
+
result.quarantinedPath = q.quarantinedPath;
|
|
1037
|
+
result.backupRef = q.backupRef;
|
|
1038
|
+
}
|
|
1027
1039
|
} catch (qErr) {
|
|
1028
1040
|
result.quarantineError = qErr.message;
|
|
1029
1041
|
log('error', `assertCleanSharedWorktree: quarantine after status-failed failed for ${worktreePath}: ${qErr.message}`);
|
|
@@ -1143,9 +1155,16 @@ async function assertCleanSharedWorktree(rootDir, worktreePath, branchName, disp
|
|
|
1143
1155
|
dirtyFiles: result.dirtyFiles,
|
|
1144
1156
|
},
|
|
1145
1157
|
);
|
|
1146
|
-
result
|
|
1147
|
-
|
|
1148
|
-
|
|
1158
|
+
// PR #3133 review — same honest-result fix as the status-failed
|
|
1159
|
+
// path above. If _quarantineDirtyWorktree was skipped by the
|
|
1160
|
+
// live-guard, do NOT set quarantined=true with a null path.
|
|
1161
|
+
if (q.skipped) {
|
|
1162
|
+
result.quarantineSkipped = true;
|
|
1163
|
+
} else {
|
|
1164
|
+
result.quarantined = true;
|
|
1165
|
+
result.quarantinedPath = q.quarantinedPath;
|
|
1166
|
+
result.backupRef = q.backupRef;
|
|
1167
|
+
}
|
|
1149
1168
|
} catch (qErr) {
|
|
1150
1169
|
result.quarantineError = qErr.message;
|
|
1151
1170
|
log('error', `assertCleanSharedWorktree: quarantine failed for ${worktreePath}: ${qErr.message}`);
|
|
@@ -1214,6 +1233,14 @@ async function _quarantineDirtyWorktree(rootDir, worktreePath, branchName, gitOp
|
|
|
1214
1233
|
|
|
1215
1234
|
// Rename the worktree dir. Once this succeeds the worktree is functionally
|
|
1216
1235
|
// quarantined; subsequent failures only affect ref bookkeeping.
|
|
1236
|
+
// W-mq5rwwss000f30a7 — never quarantine (=rename out from under) a worktree
|
|
1237
|
+
// that a live dispatch still claims. Quarantine breaks the agent's cwd just
|
|
1238
|
+
// as completely as removeWorktree would.
|
|
1239
|
+
if (shared.isWorktreePathLive(worktreePath)) {
|
|
1240
|
+
log('warn', `_quarantineDirtyWorktree: skip — live dispatch in ${worktreePath}`);
|
|
1241
|
+
shared._writeWorktreeSkipLiveInboxNote(worktreePath, '_quarantineDirtyWorktree');
|
|
1242
|
+
return { quarantinedPath: null, backupRef: null, skipped: true };
|
|
1243
|
+
}
|
|
1217
1244
|
fs.renameSync(worktreePath, quarantinedPath);
|
|
1218
1245
|
|
|
1219
1246
|
// Prune git's stale worktree metadata so the next `git worktree add` for
|
|
@@ -2047,14 +2074,14 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
2047
2074
|
const failureClassName = isDivergent ? 'WORKTREE_DIVERGENT' : 'WORKTREE_DIRTY';
|
|
2048
2075
|
const reasonMsg = cleanResult.quarantined
|
|
2049
2076
|
? `${failureClassName}: reused worktree at ${worktreePath} was dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} dirty file(s)${previewFiles ? ': ' + previewFiles : ''}) — quarantined to ${cleanResult.quarantinedPath}. Next dispatch will start fresh.`
|
|
2050
|
-
: `${failureClassName}: reused worktree at ${worktreePath} is dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} file(s)${previewFiles ? ': ' + previewFiles : ''}). Quarantine ${cleanResult.quarantineError ? 'errored: ' + cleanResult.quarantineError : 'was not attempted (' + cleanResult.reason + ').'}`;
|
|
2077
|
+
: `${failureClassName}: reused worktree at ${worktreePath} is dirty/divergent (${cleanResult.reason}; ${cleanResult.ahead || 0} ahead, ${cleanResult.behind || 0} behind, ${cleanResult.dirtyFiles?.length || 0} file(s)${previewFiles ? ': ' + previewFiles : ''}). Quarantine ${cleanResult.quarantineError ? 'errored: ' + cleanResult.quarantineError : (cleanResult.quarantineSkipped ? 'was skipped — another live dispatch claims the worktree (see notes/inbox/ engine-worktree-skip-live note).' : 'was not attempted (' + cleanResult.reason + ').')}`;
|
|
2051
2078
|
log('error', reasonMsg);
|
|
2052
2079
|
_cleanupPromptFiles();
|
|
2053
2080
|
completeDispatch(
|
|
2054
2081
|
id,
|
|
2055
2082
|
DISPATCH_RESULT.ERROR,
|
|
2056
2083
|
reasonMsg.slice(0, 500),
|
|
2057
|
-
`Engine preflight refused to dispatch into a dirty/divergent reused worktree (#2996). Reason: ${cleanResult.reason}.${cleanResult.quarantined ? ` Worktree quarantined to ${cleanResult.quarantinedPath}; backup ref ${cleanResult.backupRef || '(skipped)'}. See notes/inbox/ for recovery instructions.` : ''}`,
|
|
2084
|
+
`Engine preflight refused to dispatch into a dirty/divergent reused worktree (#2996). Reason: ${cleanResult.reason}.${cleanResult.quarantined ? ` Worktree quarantined to ${cleanResult.quarantinedPath}; backup ref ${cleanResult.backupRef || '(skipped)'}. See notes/inbox/ for recovery instructions.` : (cleanResult.quarantineSkipped ? ' Quarantine was skipped because another live dispatch claims this worktree path; this dispatch will not auto-retry until the live claimant clears.' : '')}`,
|
|
2058
2085
|
{ agentRetryable: isStatusProbeFailed && cleanResult.quarantined, failureClass: failureClassValue },
|
|
2059
2086
|
);
|
|
2060
2087
|
cleanupTempAgent(agentId);
|
|
@@ -3743,6 +3770,15 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3743
3770
|
const _projForReturn = project?.name || 'default';
|
|
3744
3771
|
const _poolSizeReturn = worktreePool.getProjectPoolSize(_projForReturn, config);
|
|
3745
3772
|
if (!_keepPidsAlive && !_managedSpawnAlive && _poolSizeReturn > 0) {
|
|
3773
|
+
// W-mq5rwwss000f30a7 — defensive live-worktree check before the
|
|
3774
|
+
// destructive `git reset --hard HEAD → git clean -fd → checkout
|
|
3775
|
+
// --detach` chain. We pass excludeDispatchId so the active row for
|
|
3776
|
+
// THIS dispatch (still in dispatch.active until completeDispatch
|
|
3777
|
+
// fires below) is ignored. Any OTHER live dispatch claiming the
|
|
3778
|
+
// same path is a sign of a concurrency bug and must not be wiped.
|
|
3779
|
+
if (shared.isWorktreePathLive(worktreePath, { excludeDispatchId: id })) {
|
|
3780
|
+
log('warn', `worktree-pool: skip return — another live dispatch claims ${worktreePath}`);
|
|
3781
|
+
} else {
|
|
3746
3782
|
try {
|
|
3747
3783
|
const _mainRefRet = sanitizeBranch(shared.resolveMainBranch(rootDir, project?.mainBranch));
|
|
3748
3784
|
await shared.shellSafeGit(['reset', '--hard', 'HEAD'], { ..._gitOpts, cwd: worktreePath, timeout: 30000 });
|
|
@@ -3771,6 +3807,7 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3771
3807
|
// dispatch-end GC below will pick it up via the isPoolMember
|
|
3772
3808
|
// check (which now correctly returns false).
|
|
3773
3809
|
}
|
|
3810
|
+
}
|
|
3774
3811
|
} else if (_keepPidsAlive || _managedSpawnAlive) {
|
|
3775
3812
|
// Skip the pool — the worktree is in use by left-running processes
|
|
3776
3813
|
// (keep_processes PIDs or managed-spawn services). Make sure no
|
|
@@ -3803,6 +3840,13 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3803
3840
|
worktreeRoot: _wtRoot,
|
|
3804
3841
|
agentId,
|
|
3805
3842
|
managedSpawnSpawnedCount: Array.isArray(managedSpawnSpawned) ? managedSpawnSpawned.length : 0,
|
|
3843
|
+
// W-mq5rwwss000f30a7 / PR #3133 review — this dispatch's own
|
|
3844
|
+
// active row still claims worktreePath (persisted at pending→active,
|
|
3845
|
+
// line ~3998). Without excludeDispatchId, shared.removeWorktree's
|
|
3846
|
+
// live-guard would skip and silently neuter the GC for the
|
|
3847
|
+
// default worktreePoolSize:0 config. Pool-return uses the same
|
|
3848
|
+
// plumbing; this is the matching wiring on the orphan-GC side.
|
|
3849
|
+
excludeDispatchId: id,
|
|
3806
3850
|
log,
|
|
3807
3851
|
});
|
|
3808
3852
|
if (_gcResult.outcome === 'gc') {
|
|
@@ -3973,6 +4017,11 @@ async function spawnAgent(dispatchItem, config) {
|
|
|
3973
4017
|
// route output parsing through the right adapter. Also surfaces the choice
|
|
3974
4018
|
// in dispatch.json for debugging multi-runtime fleets.
|
|
3975
4019
|
item.runtimeName = runtimeName;
|
|
4020
|
+
// W-mq5rwwss000f30a7 — persist the worktree path so the live-worktree
|
|
4021
|
+
// guard (shared.isWorktreePathLive) can correlate destructive callers
|
|
4022
|
+
// (removeWorktree, cleanup orphan-dir sweep, pool-return, quarantine)
|
|
4023
|
+
// back to this active dispatch and skip the wipe.
|
|
4024
|
+
if (worktreePath) item.worktreePath = worktreePath;
|
|
3976
4025
|
delete item.skipReason;
|
|
3977
4026
|
delete item._agentBusySince;
|
|
3978
4027
|
if (!dispatch.active.some(d => d.id === id)) {
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@yemi33/minions",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.2146",
|
|
4
4
|
"description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
|
|
5
5
|
"bin": {
|
|
6
6
|
"minions": "bin/minions.js"
|