@yemi33/minions 0.1.2419 → 0.1.2420

package/dashboard.js

@@ -12280,7 +12280,11 @@ What would you like to discuss or change? When you're happy, say "approve" and I
       // by a potentially-60s timeout_s. The loop self-flips state.healthy.
       const newSpec = managedSpawn.listManagedSpecs().find(s => s && s.name === name);
       if (newSpec && newSpec.healthcheck) {
-        managedSpawn.waitForFirstHealth(newSpec).catch((err) => {
+        managedSpawn.waitForFirstHealth(newSpec, { exit: runtime.exit }).then((result) => {
+          if (!result.healthy) {
+            shared.log('warn', `managed-processes restart: first health failed for ${name}: ${result.error}`);
+          }
+        }).catch((err) => {
           shared.log('warn', `managed-processes restart: waitForFirstHealth failed for ${name}: ${err.message}`);
         });
       }

package/docs/managed-spawn.md

@@ -12,7 +12,7 @@ Before managed-spawn, Constellation (and similar multi-process projects) lost de
 3. `minions restart` killed the engine but left the child running — *on Node*. **On `bun` + Windows the child died with the parent** because the Windows detached-spawn semantics differ across runtimes.
 4. Next dispatch landed against a dead URL with no signal it had died.
 
-Managed-spawn moves the spawn ownership from the agent into the engine. Agents *describe* services in a sidecar; the engine spawns them using the proven [`bin/minions.js spawnDashboard`](../bin/minions.js) pattern (works for Node, bun, python, docker, …), runs healthchecks, gates dispatch SUCCESS on first-healthy, sweeps dead PIDs / expired TTL, auto-injects live processes into downstream agent prompts, and exposes everything through `/api/managed-processes` + the dashboard.
+Managed-spawn moves the spawn ownership from the agent into the engine. Agents *describe* services in a sidecar; the engine detached-spawns an engine-owned Node launcher using the proven [`bin/minions.js spawnDashboard`](../bin/minions.js) pattern, and that launcher starts the declared executable non-detached. The launcher PID remains the managed process-tree root across engine restarts. The engine races its observed exit against first health, sweeps dead PIDs / expired TTL, auto-injects live processes into downstream agent prompts, and exposes everything through `/api/managed-processes` + the dashboard.
 
 ## When to use managed-spawn vs `keep_processes`
 
@@ -184,24 +184,25 @@ Single quotes (`'…'`) work the same way. No `powershell -Command "& '<path>' <
                                     ▼
 ┌───────────────────────────────────────────────────────────────────┐
 │ 3. spawnManagedSpec(spec) per spec                                │
-│    - Uses the proven Windows detached-spawn pattern               │
-│    - Records each in the SQL managed-process store                │
+│    - Detached Node launcher starts the target non-detached        │
+│    - Records the launcher PID in the SQL managed-process store    │
 │    - Stdio → engine/managed-logs/<name>.log (append fd, NOT pipe) │
 └───────────────────────────────────┬───────────────────────────────┘
                                     │
                                     ▼
 ┌───────────────────────────────────────────────────────────────────┐
-│ 4. waitForFirstHealth(spec) per spec, in parallel                 │
+│ 4. waitForFirstHealth(spec,{exit}) per spec, in parallel          │
+│    - Races probes against launcher/target exit                    │
 │    - Probes every interval_s, gives up at timeout_s               │
 │    - First success → state.healthy = true, last_health_at = now   │
-│    - Any failure → dispatch ERROR + sibling spawns left alive     │
-│      failure_class: managed-spawn-healthcheck (retryable)         │
+│    - Exit/timeout → dispatch ERROR + sibling spawns left alive    │
+│      failure_class: managed-spawn-healthcheck-failed              │
 └───────────────────────────────────┬───────────────────────────────┘
                                     │ all healthy
                                     ▼
 ┌───────────────────────────────────────────────────────────────────┐
 │ 5. Dispatch SUCCESS                                               │
-│    - Spec survives engine restart (PID detached + boot reconcile) │
+│    - Spec survives restart (launcher PID + boot reconcile)        │
 │    - Visible at /api/managed-processes + dashboard "Managed       │
 │      Processes" panel                                             │
 │    - Downstream agent prompts auto-inject a "## Live managed      │
@@ -287,9 +288,9 @@ All knobs live under `engine.managedSpawn` in `engine/shared.js` (`ENGINE_DEFAUL
 | Symptom | Likely cause | Fix |
 |---|---|---|
 | Dispatch ERROR `failure_class: invalid-managed-spawn` | Sidecar schema/allowlist violation | Read inbox alert; the validator includes a precise reason. Non-retryable — fix and re-dispatch. |
-| Dispatch ERROR `failure_class: managed-spawn-healthcheck` | `timeout_s` elapsed before any spec became healthy | Check `engine/managed-logs/<name>.log` for the child's crash output. Sibling spawns are left alive. Retryable. |
+| Dispatch ERROR `failure_class: managed-spawn-healthcheck-failed` | The target exited before first health, or `timeout_s` elapsed before it became healthy | Check the reported code/signal and bounded `engine/managed-logs/<name>.log` tail. Sibling spawns are left alive. |
 | WI shows yellow `⚠ managed-spawn: N/M healthy` chip on dashboard | Partial healthcheck failure: ≥1 spec passed, ≥1 failed. WI keeps `status: done` (the agent's primary work — getting an accepted sidecar — succeeded); annotation `_managedSpawnPartial = { healthy, failed[{name, reason, log_tail}], evaluated_at }` lives on the WI. Click the row to see per-spec failure reasons + last 20 log lines. Dispatch is still recorded ERROR with `failure_class: managed-spawn-healthcheck-failed`. Restart the failing spec via `POST /api/managed-processes/restart` once you've fixed the root cause (often: workspace deps not installed; see smoke-test rule in the hint). W-mpbpexrg00110661. |
-| Spec gone after `minions restart` | Bun child died with parent (the original failure mode) | Should be fixed by item 2's detached-spawn pattern. If it recurs, verify `bin/minions.js spawnDashboard` semantics still work for the runtime — that's the canonical reference. |
+| Spec gone after `minions restart` | The tracked Node launcher died or boot reconciliation dropped its PID | Inspect the managed log and verify `engine/managed-spawn-launcher.js` remained the process-tree root until the target exited. |
 | Spec listed `alive: true, healthy: false` for >30s | Healthcheck loop self-detected service degradation | The spec did not pass a subsequent healthcheck. Inspect the service; restart via API once recovered. |
 | Stale row sticks around with dead PID | Spec killed outside Minions | Wait one sweep cycle (~30 min) or call `POST /api/managed-processes/kill` manually. |
 

package/engine/managed-spawn-launcher.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+'use strict';
+
+const { spawn } = require('child_process');
+
+const [cmd, ...args] = process.argv.slice(2);
+if (!cmd) {
+  console.error('[managed-spawn-launcher] target executable missing');
+  process.exitCode = 64;
+} else {
+  let settled = false;
+  let target;
+  try {
+    target = spawn(cmd, args, {
+      cwd: process.cwd(),
+      env: process.env,
+      detached: false,
+      stdio: ['ignore', 'inherit', 'inherit'],
+      windowsHide: true,
+    });
+  } catch (error) {
+    settled = true;
+    console.error('[managed-spawn-launcher] target spawn failed: ' + error.message);
+    process.exitCode = 127;
+  }
+
+  if (target) {
+    target.once('error', (error) => {
+      if (settled) return;
+      settled = true;
+      console.error('[managed-spawn-launcher] target spawn failed: ' + error.message);
+      process.exitCode = 127;
+    });
+    target.once('exit', (code, signal) => {
+      if (settled) return;
+      settled = true;
+      console.error('[managed-spawn-launcher] target exited (code='
+        + (code == null ? 'null' : code) + ', signal=' + (signal || 'null') + ')');
+      if (signal) {
+        try {
+          process.kill(process.pid, signal);
+          return;
+        } catch {
+          process.exitCode = 1;
+          return;
+        }
+      }
+      process.exitCode = Number.isInteger(code) ? code : 1;
+    });
+  }
+}

package/engine/managed-spawn.js

@@ -29,6 +29,7 @@ const { log, ENGINE_DEFAULTS } = shared;
 
 const MANAGED_SPAWN_FILENAME = 'managed-spawn.json';
 const MANAGED_LOGS_DIR = 'managed-logs';
+const MANAGED_SPAWN_LAUNCHER_PATH = path.join(__dirname, 'managed-spawn-launcher.js');
 const INVALID_WORKDIR_REASON_PREFIX = 'invalid-workdir: ';
 
 const HEALTHCHECK_TYPES = ['http', 'command'];
@@ -777,9 +778,10 @@ function buildManagedSpawnHint(opts) {
 //                                    stdio is wired to this fd (NOT a pipe)
 //                                    so the detached process survives our
 //                                    exit on Windows.
-//   2. spawnManagedSpec(spec, ctx) → uses the proven bin/minions.js
-//                                    spawnDashboard detached-spawn pattern.
-//                                    Returns {pid, started_at, log_path}.
+//   2. spawnManagedSpec(spec, ctx) → detached-spawns an engine-owned Node
+//                                    launcher which owns the attached target.
+//                                    Returns serializable runtime facts plus
+//                                    in-memory startup/exit observations.
 //   3. recordManagedSpec(spec,     → writes one entry to
 //      runtime, ctx)                 SQL managed-process state via
 //                                    mutateJsonFileLocked. Replaces any
@@ -836,6 +838,54 @@ function _buildChildEnv(specEnv) {
   return env;
 }
 
+function _observeManagedLauncher(child) {
+  let startupSettled = false;
+  let exitSettled = false;
+  let resolveStartup;
+  let resolveExit;
+  const startup = new Promise((resolve) => { resolveStartup = resolve; });
+  const exit = new Promise((resolve) => { resolveExit = resolve; });
+
+  const settleStartup = (result) => {
+    if (startupSettled) return;
+    startupSettled = true;
+    resolveStartup(result);
+  };
+  const settleExit = (result) => {
+    if (exitSettled) return;
+    exitSettled = true;
+    resolveExit(Object.assign({ observed_at: Date.now() }, result));
+  };
+
+  child.once('spawn', () => {
+    settleStartup({ started: true, pid: child.pid, error: null });
+  });
+  child.once('error', (error) => {
+    const message = String((error && error.message) || error);
+    settleStartup({ started: false, pid: child.pid || null, error: message });
+    settleExit({ kind: 'error', code: null, signal: null, error: message });
+  });
+  child.once('exit', (code, signal) => {
+    settleStartup({ started: true, pid: child.pid, error: null });
+    settleExit({
+      kind: 'exit',
+      code: Number.isInteger(code) ? code : null,
+      signal: typeof signal === 'string' && signal ? signal : null,
+      error: null,
+    });
+  });
+  child.once('close', (code, signal) => {
+    settleExit({
+      kind: 'close',
+      code: Number.isInteger(code) ? code : null,
+      signal: typeof signal === 'string' && signal ? signal : null,
+      error: null,
+    });
+  });
+
+  return { startup, exit };
+}
+
 function spawnManagedSpec(spec, ctx) {
   if (!spec || typeof spec !== 'object') throw new Error('spawnManagedSpec: spec required');
   if (typeof spec.cmd !== 'string' || spec.cmd.length === 0) {
@@ -854,10 +904,10 @@ function spawnManagedSpec(spec, ctx) {
   const argv = Array.isArray(spec.args) ? spec.args : [];
   let child;
   try {
-    // Working Windows-correct detached spawn — same shape as
-    // bin/minions.js spawnDashboard. DO NOT switch stdio to 'pipe' or
-    // 'inherit': pipes die on EPIPE once the parent exits.
-    child = spawn(spec.cmd, argv, {
+    // Detach only process.execPath, whose Windows behavior is proven by the
+    // dashboard/engine daemon launchers. The shim starts the validated target
+    // attached and remains its process-tree root until that target exits.
+    child = spawn(process.execPath, [MANAGED_SPAWN_LAUNCHER_PATH, spec.cmd, ...argv], {
       cwd: cwd,
       env: env,
       detached: true,
@@ -873,12 +923,19 @@ function spawnManagedSpec(spec, ctx) {
   if (!child || !child.pid) {
     throw new Error('spawnManagedSpec: spawn failed for ' + spec.cmd);
   }
+  const observation = _observeManagedLauncher(child);
   child.unref();
   const startedAt = Date.now();
   log('info', 'managed-spawn born: name=' + spec.name + ' pid=' + child.pid
     + ' owner_project=' + (ctx.owner_project || '')
     + ' owner_wi=' + (ctx.owner_wi || ''));
-  return { pid: child.pid, started_at: startedAt, log_path: logPath };
+  return {
+    pid: child.pid,
+    started_at: startedAt,
+    log_path: logPath,
+    startup: observation.startup,
+    exit: observation.exit,
+  };
 }
 
 function _initialStateShape() {
@@ -1117,7 +1174,46 @@ function waitForFirstHealth(spec, opts) {
   const deadline = Date.now() + timeoutMs;
   return new Promise((resolve) => {
     let stopped = false;
+    let timer = null;
     let lastError = null;
+    const finish = (result) => {
+      if (stopped) return;
+      stopped = true;
+      if (timer) clearTimeout(timer);
+      resolve(result);
+    };
+    const exitObservation = opts.exit;
+    if (exitObservation && typeof exitObservation.then === 'function') {
+      Promise.resolve(exitObservation).then((outcome) => {
+        if (stopped) return;
+        const code = outcome && Number.isInteger(outcome.code) ? outcome.code : null;
+        const signal = outcome && typeof outcome.signal === 'string' && outcome.signal
+          ? outcome.signal : null;
+        const observerError = outcome && outcome.error ? String(outcome.error) : '';
+        let logTail = '';
+        try { logTail = (tailManagedLog(spec.name, 20) || '').slice(-2048); } catch {}
+        finish({
+          healthy: false,
+          error: 'process exited before first health (code=' + (code == null ? 'null' : code)
+            + ', signal=' + (signal || 'null') + ')'
+            + (observerError ? ': ' + observerError : ''),
+          log_tail: logTail,
+          exit: outcome || null,
+          lastCheckAt: Date.now(),
+        });
+      }, (error) => {
+        if (stopped) return;
+        let logTail = '';
+        try { logTail = (tailManagedLog(spec.name, 20) || '').slice(-2048); } catch {}
+        finish({
+          healthy: false,
+          error: 'process exit observation failed before first health (code=null, signal=null): '
+            + String((error && error.message) || error),
+          log_tail: logTail,
+          lastCheckAt: Date.now(),
+        });
+      });
+    }
     const tick = async () => {
       if (stopped) return;
       let result;
@@ -1130,26 +1226,23 @@ function waitForFirstHealth(spec, opts) {
         // hanging forever — resolve unhealthy so the caller's
         // Promise.allSettled/timeout path still progresses.
         if (stopped) return;
-        stopped = true;
-        return resolve({ healthy: false, error: String((e && e.message) || e), lastCheckAt: Date.now() });
+        return finish({ healthy: false, error: String((e && e.message) || e), lastCheckAt: Date.now() });
       }
       if (stopped) return;
       if (result.healthy) {
-        stopped = true;
         try { _markHealthy(spec.name, result.lastCheckAt); }
         catch (e) { log('warn', 'managed-spawn waitForFirstHealth: state write failed for ' + spec.name + ': ' + e.message); }
-        return resolve({ healthy: true, error: null, lastCheckAt: result.lastCheckAt });
+        return finish({ healthy: true, error: null, lastCheckAt: result.lastCheckAt });
       }
       lastError = result.error;
       if (Date.now() >= deadline) {
-        stopped = true;
-        return resolve({
+        return finish({
           healthy: false,
           error: 'timeout: spec ' + spec.name + ' did not become healthy within ' + Math.round(timeoutMs / 1000) + 's (last: ' + (lastError || 'no probes ran') + ')',
           lastCheckAt: result.lastCheckAt,
         });
       }
-      setTimeout(tick, intervalMs);
+      timer = setTimeout(tick, intervalMs);
     };
     // First probe fires immediately so a fast service doesn't pay an
     // interval_s delay.
@@ -1413,8 +1506,8 @@ function restartManagedSpec(name, opts) {
 //                                             dropped from state (no kill —
 //                                             the OS already reaped them).
 //                                          2. ttl_expires_at past now → batch
-//                                             kill via killByPidsImmediate +
-//                                             drop from state.
+//                                             tree-kill via killByPidTreesImmediate
+//                                             + drop from state.
 //                                          3. rotate log_path when size >
 //                                             logRotateBytes (rename to .1,
 //                                             overwrite any prior .1).
@@ -1474,7 +1567,7 @@ function _runManagedReconcile(opts) {
     : shared.isPidAlive;
   const killBatch = typeof opts.killBatch === 'function'
     ? opts.killBatch
-    : shared.killByPidsImmediate;
+    : shared.killByPidTreesImmediate;
   let projects;
   try {
     projects = Array.isArray(opts.projects) ? opts.projects : shared.getProjects(opts.config);
@@ -1618,7 +1711,7 @@ function removeManagedSpecsForProject(projectName) {
   }, { defaultValue: _initialStateShape() });
   let killed = 0;
   if (toKill.length > 0) {
-    try { killed = shared.killByPidsImmediate(toKill); }
+    try { killed = shared.killByPidTreesImmediate(toKill); }
     catch (e) { log('warn', 'managed-spawn removeForProject: kill batch failed for ' + projectName + ': ' + e.message); }
   }
   let unlinked = 0;

package/engine/process-utils.js

@@ -197,6 +197,38 @@ function killByPidsImmediate(pids) {
   return killed;
 }
 
+// Batched FULL-PROCESS-TREE kill — like killByPidsImmediate but walks each
+// pid's descendant tree (Windows `taskkill /F /T`; Unix killUnixProcessTree).
+// Use this when the recorded pid is a launcher/shim whose real workload runs in
+// an attached child (e.g. managed-spawn's Node launcher shim, P-8a4d6f29): a
+// non-tree kill would reap the shim and orphan/leak the child. Returns the
+// count of pids for which the kill call was issued without throwing.
+function killByPidTreesImmediate(pids) {
+  const valid = (Array.isArray(pids) ? pids : [])
+    .map(Number)
+    .filter(n => Number.isInteger(n) && n > 0 && n !== process.pid);
+  if (!valid.length) return 0;
+  if (process.platform === 'win32') {
+    const flags = valid.map(p => `/PID ${p}`).join(' ');
+    try { _execSync(`taskkill /F /T ${flags}`, { stdio: 'pipe', timeout: 5000, windowsHide: true }); return valid.length; }
+    catch {
+      // taskkill fails wholesale if ANY pid is already dead — fall back to a
+      // per-pid tree kill so live siblings still get reaped.
+      let killed = 0;
+      for (const p of valid) {
+        try { _execSync(`taskkill /PID ${p} /F /T`, { stdio: 'pipe', timeout: 3000, windowsHide: true }); killed++; }
+        catch { /* process may be dead */ }
+      }
+      return killed;
+    }
+  }
+  let killed = 0;
+  for (const p of valid) {
+    try { killUnixProcessTree(p, 'SIGKILL'); killed++; } catch { /* dead */ }
+  }
+  return killed;
+}
+
 // ─── Process Table Enumeration ───────────────────────────────────────────────
 // Cross-platform listing of every live process as { pid, ppid, name, cmd? }.
 // `cmd` is best-effort — included on Windows (via wmic) and Linux (/proc); may
@@ -822,6 +854,7 @@ module.exports = {
   killImmediate,
   killByPidImmediate,
   killByPidsImmediate,
+  killByPidTreesImmediate,
   tasklistOutputShowsPid,
   getProcessCpuSeconds,
   _parseWmicCsv,

package/engine.js

@@ -5854,7 +5854,7 @@ async function spawnAgent(dispatchItem, config) {
     // healthy:false, alive:true and the engine sweep / item-3 healthcheck
     // loop will drive its state from there.
     let managedSpawnAcceptanceFailure = null;
-    let managedSpawnSpawned = []; // [{name, pid, started_at, log_path}]
+    let managedSpawnSpawned = []; // serializable runtime facts + in-memory exit promise
     {
       const _wiMeta = dispatchItem.meta?.item?.meta || {};
       const _msEnabled = !!_wiMeta.managed_spawn
@@ -5935,7 +5935,13 @@ async function spawnAgent(dispatchItem, config) {
               try {
                 const runtime = managedSpawn.spawnManagedSpec(spec, ctx);
                 spawnedItems.push({ spec, runtime });
-                managedSpawnSpawned.push({ name: spec.name, pid: runtime.pid, started_at: runtime.started_at, log_path: runtime.log_path });
+                managedSpawnSpawned.push({
+                  name: spec.name,
+                  pid: runtime.pid,
+                  started_at: runtime.started_at,
+                  log_path: runtime.log_path,
+                  exit: runtime.exit,
+                });
               } catch (specErr) {
                 spawnFailureReason = `spawn failed for ${spec.name}: ${specErr.message}`;
                 log('warn', `managed-spawn: ${spawnFailureReason}`);
@@ -5947,7 +5953,7 @@ async function spawnAgent(dispatchItem, config) {
               // state. This is consistent with the "all healthy or fail"
               // contract item 3 will enforce on healthcheck timeout.
               for (const item of spawnedItems) {
-                try { shared.killByPidImmediate(item.runtime.pid); } catch (_e) {}
+                try { shared.killImmediate({ pid: item.runtime.pid }); } catch (_e) {}
               }
               managedSpawnSpawned = [];
               managedSpawnAcceptanceFailure = {
@@ -5999,7 +6005,8 @@ async function spawnAgent(dispatchItem, config) {
           if (!spec || !spec.healthcheck) {
             return Promise.resolve({ healthy: false, error: 'no healthcheck recorded for ' + spawned.name, _name: spawned.name });
           }
-          return managedSpawn.waitForFirstHealth(spec).then(r => Object.assign({ _name: spec.name, _pid: spawned.pid }, r));
+          return managedSpawn.waitForFirstHealth(spec, { exit: spawned.exit })
+            .then(r => Object.assign({ _name: spec.name, _pid: spawned.pid }, r));
         }));
         const failed = [];
         for (let i = 0; i < results.length; i++) {
@@ -6009,7 +6016,12 @@ async function spawnAgent(dispatchItem, config) {
             continue;
           }
           if (!r.value.healthy) {
-            failed.push({ name: r.value._name || items[i].name, pid: r.value._pid || items[i].pid, error: r.value.error || 'unhealthy' });
+            failed.push({
+              name: r.value._name || items[i].name,
+              pid: r.value._pid || items[i].pid,
+              error: r.value.error || 'unhealthy',
+              log_tail: r.value.log_tail || '',
+            });
           }
         }
         if (failed.length > 0) {
@@ -6024,8 +6036,8 @@ async function spawnAgent(dispatchItem, config) {
           // re-reading the log files later. Use the same tailManagedLog helper
           // the inbox alert uses; cap each tail to 2KB to bound the WI write.
           const _failedDetails = failed.map(f => {
-            let tail = '';
-            try { tail = (managedSpawn.tailManagedLog(f.name, 20) || '').slice(-2048); }
+            let tail = f.log_tail || '';
+            try { if (!tail) tail = (managedSpawn.tailManagedLog(f.name, 20) || '').slice(-2048); }
             catch (_e) { tail = ''; }
             return { name: f.name, reason: f.error, log_tail: tail };
           });
@@ -6051,7 +6063,7 @@ async function spawnAgent(dispatchItem, config) {
           try {
             const wiId = dispatchItem.meta?.item?.id || '';
             const logTails = failed.map(f => {
-              const tail = managedSpawn.tailManagedLog(f.name, 50) || '(log empty or unreadable)';
+              const tail = f.log_tail || managedSpawn.tailManagedLog(f.name, 50) || '(log empty or unreadable)';
               return '### ' + f.name + ' (pid ' + (f.pid || '?') + ')\n\nReason: `' + f.error + '`\n\n```\n' + tail.slice(-2000) + '\n```';
             }).join('\n\n');
             const alertBody = [

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@yemi33/minions",
-  "version": "0.1.2419",
+  "version": "0.1.2420",
   "description": "Multi-agent AI dev team that runs from ~/.minions/ — five autonomous agents share a single engine, dashboard, and knowledge base",
   "bin": {
     "minions": "bin/minions.js"