@yemi33/minions 0.1.2144 → 0.1.2145

package/engine/watchdog.js

@@ -0,0 +1,458 @@
+// engine/watchdog.js — out-of-process recovery scheduled by the OS.
+//
+// PROBLEM (cluster-kill):
+//   The in-process supervisor (engine/supervisor.js) respawns engine/dashboard
+//   when one of them dies in isolation, but it can't respawn ITSELF. When
+//   something kills all three at once — Windows job-object teardown when a
+//   parent terminal closes (RDP disconnect, Windows Terminal auto-update,
+//   manual close), Linux OOM-killer, systemd cgroup memory limits, manual
+//   `pkill -f node`, system reboot — recovery requires an external agent.
+//
+// SOLUTION:
+//   Register `minions watchdog tick` with the OS scheduler (Task Scheduler /
+//   launchd / systemd --user) to run every N minutes. Each tick probes the
+//   stack, logs the verdict, and invokes `minions start`/`restart` if dead.
+//   The scheduler itself lives outside any user terminal's job/session tree,
+//   so it survives every cluster-kill scenario that takes minions down.
+//
+// CONTRACT:
+//   tick(opts)                → always resolves; never throws to the caller.
+//                              The scheduler must never see a non-zero exit,
+//                              or it may start escalating (eventvwr, syslog,
+//                              email-on-failure) for the very recovery we're
+//                              trying to make boring.
+//   install(opts)             → throws on platform-tool failure (so the user
+//                              sees it; install is interactive).
+//   uninstall()               → idempotent; removing a non-existent
+//                              registration is success, not error.
+//   status()                  → returns { installed, scheduler, … details }.
+//
+// Cross-platform note: install/uninstall/status branch on process.platform.
+// Each platform's helper is self-contained so unit tests can stub spawnSync.
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const { spawn, spawnSync } = require('child_process');
+
+const DEFAULT_INTERVAL_MIN = 5;
+const DEFAULT_DASH_PORT = 7331;
+const WIN_TASK_NAME = 'MinionsWatchdog';
+const MAC_PLIST_LABEL = 'io.minions.watchdog';
+const LINUX_UNIT_NAME = 'minions-watchdog';
+const TICK_SPAWN_TIMEOUT_MS = 120000;
+const WATCHDOG_LOG_NAME = 'watchdog-stdio.log';
+// Cap the watchdog log so a misbehaving scheduler can't fill the disk
+// over months. Truncates to half-size on overflow (oldest lines dropped).
+const WATCHDOG_LOG_MAX_BYTES = 2 * 1024 * 1024;
+
+function logLine(minionsHome, msg) {
+  try {
+    const dir = path.join(minionsHome, 'engine');
+    fs.mkdirSync(dir, { recursive: true });
+    const logPath = path.join(dir, WATCHDOG_LOG_NAME);
+    try {
+      const stat = fs.statSync(logPath);
+      if (stat.size > WATCHDOG_LOG_MAX_BYTES) {
+        const buf = fs.readFileSync(logPath);
+        const tail = buf.subarray(Math.floor(buf.length / 2));
+        const firstNewline = tail.indexOf(0x0a);
+        const trimmed = firstNewline >= 0 ? tail.subarray(firstNewline + 1) : tail;
+        fs.writeFileSync(logPath, trimmed);
+      }
+    } catch { /* file may not exist yet — that's fine */ }
+    fs.appendFileSync(logPath, `[${new Date().toISOString()}] ${msg}\n`);
+  } catch { /* logging failure must never escalate to the scheduler */ }
+}
+
+function isPidAlive(pid) {
+  if (!pid || !Number.isInteger(pid) || pid <= 0) return false;
+  try { process.kill(pid, 0); return true; } catch { return false; }
+}
+
+/**
+ * Probe engine + dashboard and recover if needed. Always resolves (never
+ * throws). Returns { healthy, action, exitCode? } describing what was done.
+ *
+ * Recovery policy:
+ *   - stop-intent flag set      → STAND DOWN (user wanted minions stopped;
+ *                                respawning would undo `minions stop`)
+ *   - both alive                → no-op
+ *   - both dead                 → `minions start` (idempotent fast path)
+ *   - one alive one dead        → `minions restart` (only restart can
+ *                                reconcile partial state — start refuses
+ *                                by design)
+ */
+async function tick(opts) {
+  opts = opts || {};
+  const minionsHome = opts.minionsHome;
+  const minionsBin = opts.minionsBin;
+  const dashPort = opts.dashPort || DEFAULT_DASH_PORT;
+  const readEnginePid = opts.readEnginePid;
+  const isPortListening = opts.isPortListening;
+  const isStopIntentSet = opts.isStopIntentSet;
+  const spawner = opts.spawner || spawn;
+  const now = opts.now || (() => new Date());
+
+  if (!minionsHome || !minionsBin || !readEnginePid || !isPortListening) {
+    // Programmer error — caller forgot a dep. Try to log when minionsHome is
+    // available (logLine swallows errors of its own), then resolve with a
+    // marker. The scheduler must never see a non-zero exit.
+    if (minionsHome) logLine(minionsHome, 'missing-deps → noop (caller did not provide all required helpers)');
+    return { healthy: false, action: 'noop', error: 'missing-deps' };
+  }
+
+  // Respect `minions stop` / `minions uninstall` / mid-restart windows.
+  // On forks that ship an in-process supervisor (yemi33/minions), this flag
+  // is the same one the supervisor uses to suppress engine respawns; the
+  // watchdog MUST honor it too, or it would brain-deadly restart minions
+  // every N minutes after the user explicitly stopped it.
+  // On forks that DON'T ship a supervisor + stop-intent producer (e.g.
+  // opg-microsoft/minions), `isStopIntentSet` is intentionally absent — we
+  // treat that as "no stop-intent system, proceed with recovery" (fail-open).
+  let stopWanted = false;
+  if (typeof isStopIntentSet === 'function') {
+    try { stopWanted = !!isStopIntentSet(); } catch { stopWanted = false; }
+  }
+  if (stopWanted) {
+    logLine(minionsHome, `stop-intent set → standing down (no probe, no recovery)`);
+    return { healthy: false, action: 'stand-down' };
+  }
+
+  let enginePid = null;
+  try { enginePid = readEnginePid(minionsHome); } catch { enginePid = null; }
+  const engineAlive = isPidAlive(enginePid);
+  let dashUp = false;
+  try { dashUp = !!isPortListening(dashPort); } catch { dashUp = false; }
+
+  if (engineAlive && dashUp) {
+    logLine(minionsHome, `ok engine=${enginePid} port=${dashPort} ts=${now().toISOString()}`);
+    return { healthy: true, action: 'none' };
+  }
+
+  const partial = engineAlive || dashUp;
+  const action = partial ? 'restart' : 'start';
+  logLine(
+    minionsHome,
+    `unhealthy engine=${enginePid || '-'}(${engineAlive ? 'alive' : 'dead'}) ` +
+    `port=${dashPort}(${dashUp ? 'up' : 'down'}) → invoking minions ${action}`
+  );
+
+  try {
+    // Detach the child so its lifetime is independent of this tick process.
+    // The OS scheduler closes our stdio shortly after exit; we don't want
+    // that closure to cascade into the new daemon stack.
+    const child = spawner(process.execPath, [minionsBin, action], {
+      detached: true,
+      stdio: 'ignore',
+      windowsHide: true,
+    });
+    if (child && typeof child.unref === 'function') child.unref();
+    logLine(minionsHome, `spawned minions ${action} pid=${child && child.pid} (detached)`);
+    return { healthy: false, action, spawnedPid: child && child.pid };
+  } catch (err) {
+    logLine(minionsHome, `FAILED to spawn minions ${action}: ${err && err.message || err}`);
+    return { healthy: false, action, error: String(err && err.message || err) };
+  }
+}
+// Documented ceiling for any future synchronous variant of tick. Not used
+// by the detached spawn path, but kept reachable so callers can opt in.
+tick.TICK_SPAWN_TIMEOUT_MS = TICK_SPAWN_TIMEOUT_MS;
+
+// ─── Install / uninstall / status ────────────────────────────────────────────
+
+function install(opts) {
+  opts = opts || {};
+  const plat = opts.platform || process.platform;
+  const minionsBin = opts.minionsBin;
+  const intervalMin = Number.isFinite(opts.intervalMin) && opts.intervalMin > 0
+    ? Math.floor(opts.intervalMin)
+    : DEFAULT_INTERVAL_MIN;
+  const nodeBin = opts.nodeBin || process.execPath;
+  if (!minionsBin) throw new Error('install: minionsBin is required');
+
+  if (plat === 'win32') return installWindows({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
+  if (plat === 'darwin') return installMac({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
+  if (plat === 'linux') return installLinux({ nodeBin, minionsBin, intervalMin, minionsHome: opts.minionsHome, spawner: opts.spawner });
+  throw new Error(`watchdog install: unsupported platform "${plat}"`);
+}
+
+function uninstall(opts) {
+  opts = opts || {};
+  const plat = opts.platform || process.platform;
+  if (plat === 'win32') return uninstallWindows({ minionsHome: opts.minionsHome, spawner: opts.spawner });
+  if (plat === 'darwin') return uninstallMac({ spawner: opts.spawner });
+  if (plat === 'linux') return uninstallLinux({ spawner: opts.spawner });
+  throw new Error(`watchdog uninstall: unsupported platform "${plat}"`);
+}
+
+function status(opts) {
+  opts = opts || {};
+  const plat = opts.platform || process.platform;
+  if (plat === 'win32') return statusWindows({ spawner: opts.spawner });
+  if (plat === 'darwin') return statusMac({ spawner: opts.spawner });
+  if (plat === 'linux') return statusLinux({ spawner: opts.spawner });
+  throw new Error(`watchdog status: unsupported platform "${plat}"`);
+}
+
+// ─── Windows: Task Scheduler ─────────────────────────────────────────────────
+
+// On Windows we generate a launcher .cmd in MINIONS_HOME/engine/ that sets
+// MINIONS_HOME before invoking node, then point schtasks at that file. This
+// avoids two failure modes that the obvious schtasks /TR "node bin tick"
+// form has:
+//   (a) The task runs without MINIONS_HOME in env (Task Scheduler doesn't
+//       inherit the install-time user session env), so a user with a
+//       non-default MINIONS_HOME would have the watchdog probe and "recover"
+//       the WRONG stack (~/.minions instead of their real install).
+//   (b) /TR has hard length + quoting limits for nested arguments. A
+//       launcher script keeps schtasks's /TR simple and gives the user a
+//       readable, auditable file to inspect on disk.
+function buildWindowsLauncher(opts) {
+  return `@echo off\r
+REM Minions watchdog launcher — auto-generated by 'minions watchdog install'.\r
+REM Re-run 'minions watchdog install' to regenerate after moving MINIONS_HOME.\r
+set "MINIONS_HOME=${opts.minionsHome}"\r
+"${opts.nodeBin}" "${opts.minionsBin}" watchdog tick\r
+`;
+}
+
+function windowsLauncherPath(minionsHome) {
+  return path.join(minionsHome, 'engine', 'watchdog-launcher.cmd');
+}
+
+function installWindows(opts) {
+  const sp = opts.spawner || spawnSync;
+  if (!opts.minionsHome) throw new Error('install (win32): minionsHome is required for launcher path');
+  const launcherPath = windowsLauncherPath(opts.minionsHome);
+  fs.mkdirSync(path.dirname(launcherPath), { recursive: true });
+  fs.writeFileSync(launcherPath, buildWindowsLauncher(opts));
+  // /TR points at the launcher. Quote it so spaces in MINIONS_HOME survive.
+  const tr = `"${launcherPath}"`;
+  const args = [
+    '/Create', '/F',
+    '/SC', 'MINUTE', '/MO', String(opts.intervalMin),
+    '/TN', WIN_TASK_NAME,
+    '/TR', tr,
+    '/RL', 'LIMITED',
+  ];
+  const r = sp('schtasks.exe', args, { encoding: 'utf8', windowsHide: true });
+  if (r.status !== 0) {
+    throw new Error(`schtasks /Create failed (exit ${r.status}): ${(r.stderr || r.stdout || '').trim()}`);
+  }
+  return { ok: true, scheduler: 'Task Scheduler', taskName: WIN_TASK_NAME, intervalMin: opts.intervalMin, launcherPath };
+}
+
+function uninstallWindows(opts) {
+  const sp = opts.spawner || spawnSync;
+  const r = sp('schtasks.exe', ['/Delete', '/F', '/TN', WIN_TASK_NAME], { encoding: 'utf8', windowsHide: true });
+  // Exit 1 from schtasks usually means "task not found" — idempotent success.
+  const removed = r.status === 0;
+  // Best-effort launcher cleanup. Caller may pass minionsHome (so we know
+  // where to look); when omitted (e.g. CLI doesn't thread it through), we
+  // skip the file deletion — it's harmless to leave behind.
+  if (opts.minionsHome) {
+    try { fs.unlinkSync(windowsLauncherPath(opts.minionsHome)); } catch { /* ok */ }
+  }
+  return { ok: true, scheduler: 'Task Scheduler', taskName: WIN_TASK_NAME, removed };
+}
+
+function statusWindows(opts) {
+  const sp = opts.spawner || spawnSync;
+  const r = sp('schtasks.exe', ['/Query', '/TN', WIN_TASK_NAME, '/V', '/FO', 'LIST'], { encoding: 'utf8', windowsHide: true });
+  return {
+    installed: r.status === 0,
+    scheduler: 'Task Scheduler',
+    taskName: WIN_TASK_NAME,
+    details: (r.stdout || r.stderr || '').trim(),
+  };
+}
+
+// ─── macOS: launchd LaunchAgent ──────────────────────────────────────────────
+
+function macPlistPath() {
+  return path.join(os.homedir(), 'Library', 'LaunchAgents', `${MAC_PLIST_LABEL}.plist`);
+}
+
+function buildMacPlist(opts) {
+  const stdoutPath = path.join(opts.minionsHome, 'engine', 'watchdog-launchd.log');
+  return `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key><string>${MAC_PLIST_LABEL}</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>${opts.nodeBin}</string>
+    <string>${opts.minionsBin}</string>
+    <string>watchdog</string>
+    <string>tick</string>
+  </array>
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>MINIONS_HOME</key><string>${opts.minionsHome}</string>
+  </dict>
+  <key>StartInterval</key><integer>${opts.intervalMin * 60}</integer>
+  <key>RunAtLoad</key><true/>
+  <key>StandardOutPath</key><string>${stdoutPath}</string>
+  <key>StandardErrorPath</key><string>${stdoutPath}</string>
+</dict>
+</plist>
+`;
+}
+
+function installMac(opts) {
+  const sp = opts.spawner || spawnSync;
+  if (!opts.minionsHome) throw new Error('install (mac): minionsHome is required for log path and env capture');
+  const plistPath = macPlistPath();
+  fs.mkdirSync(path.dirname(plistPath), { recursive: true });
+  fs.writeFileSync(plistPath, buildMacPlist(opts));
+  // Unload first (ignore failure — agent may not be loaded) then load to
+  // pick up changes. Use legacy `load -w` for broad compatibility across
+  // 10.10+; modern `bootstrap`/`bootout` would require parsing `id -u`.
+  sp('launchctl', ['unload', plistPath], { stdio: 'ignore' });
+  const r = sp('launchctl', ['load', '-w', plistPath], { encoding: 'utf8' });
+  if (r.status !== 0) {
+    throw new Error(`launchctl load failed (exit ${r.status}): ${(r.stderr || r.stdout || '').trim()}`);
+  }
+  return { ok: true, scheduler: 'launchd', plistPath, intervalMin: opts.intervalMin };
+}
+
+function uninstallMac(opts) {
+  const sp = opts.spawner || spawnSync;
+  const plistPath = macPlistPath();
+  sp('launchctl', ['unload', plistPath], { stdio: 'ignore' });
+  let removed = false;
+  try { fs.unlinkSync(plistPath); removed = true; } catch { /* missing = idempotent */ }
+  return { ok: true, scheduler: 'launchd', plistPath, removed };
+}
+
+function statusMac(opts) {
+  const sp = opts.spawner || spawnSync;
+  const plistPath = macPlistPath();
+  const installed = fs.existsSync(plistPath);
+  const r = sp('launchctl', ['list', MAC_PLIST_LABEL], { encoding: 'utf8' });
+  return {
+    installed,
+    scheduler: 'launchd',
+    plistPath,
+    loaded: r.status === 0,
+    details: (r.stdout || r.stderr || '').trim(),
+  };
+}
+
+// ─── Linux: systemd --user timer ─────────────────────────────────────────────
+
+function linuxUnitDir() {
+  return path.join(os.homedir(), '.config', 'systemd', 'user');
+}
+
+function buildLinuxService(opts) {
+  // Quote both paths so systemd's whitespace-tokenizing ExecStart parser
+  // doesn't split a path containing spaces (e.g. a Nix-style /nix/store
+  // path or a custom install under /opt/My Tools/). systemd.exec(5) supports
+  // double-quoted args; backslashes inside become literal so paths are safe.
+  const execStart = `"${opts.nodeBin}" "${opts.minionsBin}" watchdog tick`;
+  // Pin MINIONS_HOME for the same reason as Windows/Mac: systemd user units
+  // run in a stripped env that won't carry the user's interactive MINIONS_HOME
+  // override.
+  return `[Unit]
+Description=Minions watchdog — probe + heal the engine/dashboard stack
+
+[Service]
+Type=oneshot
+Environment="MINIONS_HOME=${opts.minionsHome}"
+ExecStart=${execStart}
+`;
+}
+
+function buildLinuxTimer(opts) {
+  return `[Unit]
+Description=Minions watchdog timer (every ${opts.intervalMin} min)
+
+[Timer]
+OnBootSec=1min
+OnUnitActiveSec=${opts.intervalMin}min
+Unit=${LINUX_UNIT_NAME}.service
+
+[Install]
+WantedBy=timers.target
+`;
+}
+
+function installLinux(opts) {
+  const sp = opts.spawner || spawnSync;
+  if (!opts.minionsHome) throw new Error('install (linux): minionsHome is required for Environment= line');
+  const unitDir = linuxUnitDir();
+  fs.mkdirSync(unitDir, { recursive: true });
+  fs.writeFileSync(path.join(unitDir, `${LINUX_UNIT_NAME}.service`), buildLinuxService(opts));
+  fs.writeFileSync(path.join(unitDir, `${LINUX_UNIT_NAME}.timer`), buildLinuxTimer(opts));
+  sp('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' });
+  const r = sp('systemctl', ['--user', 'enable', '--now', `${LINUX_UNIT_NAME}.timer`], { encoding: 'utf8' });
+  if (r.status !== 0) {
+    const userName = os.userInfo().username;
+    throw new Error(
+      `systemctl --user enable --now ${LINUX_UNIT_NAME}.timer failed (exit ${r.status}): ` +
+      `${(r.stderr || r.stdout || '').trim()}\n` +
+      `Hint: if you're not logged into a graphical session, run ` +
+      `'sudo loginctl enable-linger ${userName}' so the user manager survives logout, ` +
+      `then retry 'minions watchdog install'.`
+    );
+  }
+  return {
+    ok: true,
+    scheduler: 'systemd --user',
+    servicePath: path.join(unitDir, `${LINUX_UNIT_NAME}.service`),
+    timerPath: path.join(unitDir, `${LINUX_UNIT_NAME}.timer`),
+    intervalMin: opts.intervalMin,
+    lingerHint: `Run 'sudo loginctl enable-linger ${os.userInfo().username}' so the timer survives logout.`,
+  };
+}
+
+function uninstallLinux(opts) {
+  const sp = opts.spawner || spawnSync;
+  sp('systemctl', ['--user', 'disable', '--now', `${LINUX_UNIT_NAME}.timer`], { stdio: 'ignore' });
+  const unitDir = linuxUnitDir();
+  let removed = false;
+  for (const f of [`${LINUX_UNIT_NAME}.timer`, `${LINUX_UNIT_NAME}.service`]) {
+    try { fs.unlinkSync(path.join(unitDir, f)); removed = true; } catch { /* idempotent */ }
+  }
+  sp('systemctl', ['--user', 'daemon-reload'], { stdio: 'ignore' });
+  return { ok: true, scheduler: 'systemd --user', removed };
+}
+
+function statusLinux(opts) {
+  const sp = opts.spawner || spawnSync;
+  const timerPath = path.join(linuxUnitDir(), `${LINUX_UNIT_NAME}.timer`);
+  const installed = fs.existsSync(timerPath);
+  const r = sp('systemctl', ['--user', 'status', `${LINUX_UNIT_NAME}.timer`, '--no-pager'], { encoding: 'utf8' });
+  return {
+    installed,
+    scheduler: 'systemd --user',
+    timerPath,
+    active: r.status === 0,
+    details: (r.stdout || r.stderr || '').trim(),
+  };
+}
+
+module.exports = {
+  tick,
+  install,
+  uninstall,
+  status,
+  isPidAlive,
+  // Constants + builders exported for unit tests and callers wiring CLI help.
+  DEFAULT_INTERVAL_MIN,
+  DEFAULT_DASH_PORT,
+  WIN_TASK_NAME,
+  MAC_PLIST_LABEL,
+  LINUX_UNIT_NAME,
+  WATCHDOG_LOG_NAME,
+  buildMacPlist,
+  buildLinuxService,
+  buildLinuxTimer,
+  buildWindowsLauncher,
+  macPlistPath,
+  windowsLauncherPath,
+  linuxUnitDir,
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2144",
+  "version": "0.1.2145",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"