@yemi33/minions 0.1.2448 → 0.1.2449

package/engine/recovery/stop-stack.js

@@ -0,0 +1,607 @@
+// engine/recovery/stop-stack.js
+//
+// P-a7f3c91d — the SINGLE owner of ordered whole-stack runtime teardown.
+//
+// Why this module exists
+// ----------------------
+// There was no whole-stack stop. `minions stop` is delegated straight to
+// `engine.js stop`, and engine/operations/cli.js#stop() only (a) writes
+// stop-intent and (b) stamps `_shutdownRequestedAt` into control.json before
+// returning. It never touches the dashboard or the supervisor. Because the
+// dashboard independently holds engine/state.db open, `engine/state.db-shm`
+// stays present and non-empty after a "successful" stop and every quiescence
+// probe correctly stays red — which is exactly what blocked the internal
+// installer's pre-migration gate.
+//
+// The ordered sequence below is NOT new machinery. It already existed inline
+// twice in the CLI (the `restart` path, and abbreviated in `uninstall`); this
+// module extracts that proven order into one importable, injectable owner:
+//
+//   1. write stop-intent FIRST  — so the supervisor stands down before
+//      anything dies and cannot race-respawn what we are about to stop
+//   2. stop the supervisor      — before the daemons, so its guard tick can't
+//                                 fire against a half-dead stack
+//   3. stop the dashboard       — the second holder of engine/state.db
+//   4. request the engine's graceful drain (`node engine.js stop`)
+//   5. wait for the engine PID to die inside the drain budget
+//   6. identity-verified reap of whatever survived
+//   7. wait for engine/state.db-shm to be released
+//   8. ONE late-respawn sweep to close the supervisor/watchdog race window,
+//      then the final -shm verification
+//
+// Deliberate constraints
+// ----------------------
+// * bin/minions.js has NO `module.exports` — it is a CLI entry script, not an
+//   importable module. Nothing here may import from it. This module owns the
+//   sequence and builds on engine/core/shared.js (which re-exports the
+//   engine/processes/process-utils.js termination gateway); the CLI becomes a
+//   consumer, not a dependency.
+// * ALL termination routes through the registered `shared.terminateProcess`
+//   site `bin.daemon-stop-process` — mode `process-only`, ownership
+//   `exact-scripts`. Process-only is load-bearing on BOTH platforms: detached
+//   cold agents are children of the engine and MUST survive so the replacement
+//   engine can reattach them. That is why the CLI restart path uses
+//   killPidOnly rather than a graceful tree kill, and it is preserved here.
+// * Root scoping: every candidate PID is proven against daemon script paths
+//   rooted under the SUPPLIED `minionsHome`. A second install, a `--dev`
+//   checkout, or an unrelated node process is never signalled. A live PID we
+//   cannot prove owns this root yields a REFUSAL verdict, not a kill.
+// * Never throws for an expected outcome (still alive, unverified holder,
+//   budget expired) — callers get a structured verdict.
+// * Every process seam and the clock are injectable through `deps`, so the
+//   whole sequence is asserted without timing sleeps.
+//
+// Synchronous by design: both consumers (the CLI teardown paths and the
+// spawnSync-only internal installer) run inside straight-line shutdown code.
+
+const fs = require('fs');
+const path = require('path');
+const { execFileSync } = require('child_process');
+const shared = require('../core/shared');
+
+/** The ordered steps recorded on every verdict. Exported so consumers and
+ *  tests assert ordering against names rather than array positions. */
+const STOP_STACK_STEPS = Object.freeze({
+  STOP_INTENT: 'stop-intent',
+  SUPERVISOR: 'supervisor',
+  DASHBOARD: 'dashboard',
+  ENGINE_DRAIN: 'engine-drain',
+  ENGINE_WAIT: 'engine-wait',
+  REAP: 'reap',
+  SHM_WAIT: 'shm-wait',
+  LATE_SWEEP: 'late-sweep',
+  VERIFY: 'verify',
+});
+
+/**
+ * Why a PID ended up in `holders[]`. The distinction is load-bearing:
+ * `UNVERIFIED_IDENTITY` means the process table WAS readable and proved the PID
+ * is not one of our daemons — a stale PID file, not a running stack. The other
+ * two mean something of ours is (or may still be) alive.
+ */
+const STOP_STACK_HOLDER_REASONS = Object.freeze({
+  UNVERIFIED_IDENTITY: 'unverified-identity',
+  IDENTITY_PROBE_FAILED: 'identity-probe-failed',
+  STILL_ALIVE: 'still-alive',
+});
+
+/**
+ * Machine-readable exit codes for callers that shell out to a whole-stack stop
+ * (`minions stop --all`, and the internal installer's pre-migration gate).
+ *
+ * They exist so a caller can BRANCH without scraping stdout, and they are
+ * deliberately not 1 — the CLI already uses 1 for generic failures, so a
+ * distinct code is the only way to tell "the runtime is still busy, retry or
+ * widen the budget" apart from "Minions refused to signal a process it could
+ * not prove is yours; a human must look".
+ */
+const STOP_STACK_EXIT = Object.freeze({
+  OK: 0,
+  /** The ordered teardown ran, but the runtime was not quiesced when the
+   *  budget expired (a daemon still alive, or the WAL index still held). */
+  NOT_QUIESCED: 3,
+  /** Nothing was force-killed because identity could not be proven. */
+  REFUSED_UNVERIFIED: 4,
+});
+
+// Matches the internal installer's quiescence window so a stop driven by the
+// migration gate cannot be shorter than the wait that follows it.
+const DEFAULT_STOP_TIMEOUT_MS = 60000;
+const PID_POLL_MS = 100;
+const SHM_POLL_MS = 500;
+/**
+ * Budget-INDEPENDENT floor for every wait that follows a kill we already issued.
+ *
+ * The forced platform kill is asynchronous on both Windows and POSIX: the call
+ * returns once the signal is ACCEPTED, and the process leaves the table some
+ * milliseconds later. The forced reap exists precisely for a hung engine — and
+ * in that case the graceful drain (step 5) has already spent the entire caller
+ * budget, so a post-kill wait derived from the remaining budget is exactly 0. It
+ * would return instantly, the final verification would see a process that is
+ * already on its way out, and the verdict would report it as a surviving
+ * holder: `minions restart` aborts with the stack DOWN, and `stop --all --wait`
+ * reports NOT_QUIESCED moments after a successful reap.
+ *
+ * Once the signal is out, abandoning it mid-flight buys nothing — the process
+ * is dying either way, and the only thing a zero wait changes is the honesty of
+ * the verdict. This mirrors the budget-independent 5000ms grace the inline CLI
+ * code used before this module extracted it (bin/minions.js: the post-kill
+ * `waitForPidDeath(oldEnginePid, 5000)` and the late-respawn sweep's
+ * `waitForPidDeath(pid, 5000)`), and it is a FLOOR, never a cap: a caller with
+ * budget left still gets its full remaining budget.
+ */
+const POST_KILL_GRACE_MS = 5000;
+// Budget for the `engine.js stop` CALL itself. The call only stamps
+// control.json and returns; the drain is awaited separately by ENGINE_WAIT.
+const ENGINE_STOP_CALL_TIMEOUT_MS = 10000;
+
+const STATE_SHM_SEGMENTS = ['engine', 'state.db-shm'];
+
+/**
+ * The long-lived daemons this module owns, in stop order, with the PID file
+ * each one writes and the script that proves its identity.
+ *
+ * Agent processes are deliberately absent: they are detached, they must
+ * survive for restart reattachment, and any database handle they hold is still
+ * caught by the -shm evidence below.
+ */
+const SERVICE_DEFS = Object.freeze([
+  { name: 'engine', pidFile: ['engine', 'control.json'], shape: 'json', script: ['engine.js'] },
+  { name: 'dashboard', pidFile: ['engine', 'dashboard-port.json'], shape: 'json', script: ['dashboard.js'] },
+  { name: 'supervisor', pidFile: ['engine', 'supervisor.pid'], shape: 'text', script: ['engine', 'recovery', 'supervisor.js'] },
+]);
+
+/** Daemon script paths rooted under `minionsHome` — the identity allowlist. */
+function daemonScriptPathsForRoot(minionsHome) {
+  const root = path.resolve(String(minionsHome || ''));
+  return SERVICE_DEFS.map(def => path.join(root, ...def.script));
+}
+
+/**
+ * Recorded service PIDs for `minionsHome`, each carrying the script path that
+ * must prove its identity before it may be signalled.
+ *
+ * Read from the files the daemons themselves write, so this works with the
+ * dashboard already down — which is exactly when a stop is most likely to be
+ * attempted. (bin/install-internal-minions.js#readServicePids reads the same
+ * three files for its read-only quiescence probe; this variant additionally
+ * resolves the owning script, which is what makes termination safe.)
+ */
+function readServiceRecords(minionsHome, deps = {}) {
+  const root = path.resolve(String(minionsHome || ''));
+  const readFileSync = typeof deps.readFileSync === 'function'
+    ? deps.readFileSync
+    : (p => fs.readFileSync(p, 'utf8'));
+  return SERVICE_DEFS.map((def) => {
+    const file = path.join(root, ...def.pidFile);
+    let pid = null;
+    try {
+      const raw = String(readFileSync(file));
+      pid = def.shape === 'json' ? Number(JSON.parse(raw).pid) : Number(raw.trim());
+    } catch { pid = null; }
+    return {
+      name: def.name,
+      pid: Number.isInteger(pid) && pid > 0 ? pid : null,
+      file,
+      script: path.join(root, ...def.script),
+    };
+  });
+}
+
+function defaultWriteStopIntent(minionsHome, source) {
+  const target = path.join(path.resolve(String(minionsHome || '')), 'engine', 'stop-intent.json');
+  // Prefer the shared writer when it targets the same root, so the flag stays
+  // byte-identical to every other stop path. A pinned foreign root (the
+  // installer's case) falls through to a root-scoped write.
+  try {
+    if (path.resolve(shared.STOP_INTENT_PATH) === path.resolve(target)) {
+      return shared.writeStopIntent(source) !== false;
+    }
+  } catch { /* fall through to the explicit write */ }
+  try {
+    fs.mkdirSync(path.dirname(target), { recursive: true });
+    fs.writeFileSync(target, JSON.stringify({
+      stoppedAt: new Date().toISOString(),
+      source: source || 'stopRuntimeStack',
+    }, null, 2));
+    return true;
+  } catch { return false; }
+}
+
+/**
+ * Force-stop one PID without walking its descendants, and only after the
+ * gateway has proven the PID runs one of `scriptPaths`. `execFileSync` is not
+ * used here — the registered site owns the platform specifics.
+ */
+function defaultKillPidOnly(pid, scriptPaths) {
+  const n = Number(pid);
+  if (!Number.isInteger(n) || n <= 0 || n === process.pid) return false;
+  return shared.terminateProcess('bin.daemon-stop-process', { pid: n }, { scriptPaths });
+}
+
+/** Ask the engine to drain gracefully. Mirrors the CLI restart path, but via
+ *  execFileSync so a runtime root containing spaces needs no shell quoting.
+ *  The `_execFileSync` seam exists only so the default env-scoping can be
+ *  asserted directly without spawning a real engine. */
+function defaultRequestEngineStop({ minionsHome, timeoutMs }, _execFileSync = execFileSync) {
+  const root = path.resolve(String(minionsHome || ''));
+  // Pin the child engine onto the SAME root every other seam here is scoped to.
+  // resolveMinionsHome() returns MINIONS_TEST_DIR, then MINIONS_HOME, BEFORE any
+  // root-pointer logic (engine/core/shared.js), so without this the spawned
+  // `engine.js stop` would resolve MINIONS_DIR from whatever ambient env it
+  // inherited — draining/rewriting an UNRELATED root's control.json while the
+  // pinned root's engine never drains and is force-reaped. MINIONS_TEST_DIR
+  // outranks MINIONS_HOME, so an inherited one must be dropped or it silently
+  // un-pins the child (mirrors bin/install-internal-minions.js:1992-1995).
+  const childEnv = { ...process.env, MINIONS_HOME: root };
+  delete childEnv.MINIONS_TEST_DIR;
+  try {
+    _execFileSync(process.execPath, [path.join(root, 'engine.js'), 'stop'], {
+      stdio: 'ignore',
+      cwd: root,
+      env: childEnv,
+      timeout: Math.max(1, Number(timeoutMs) || ENGINE_STOP_CALL_TIMEOUT_MS),
+      windowsHide: true,
+    });
+    return { ok: true };
+  } catch (err) {
+    return { ok: false, error: String((err && err.message) || err) };
+  }
+}
+
+function resolveDeps(deps = {}) {
+  const listPlural = typeof deps.listNodePidsMatchingScripts === 'function'
+    ? deps.listNodePidsMatchingScripts
+    : (typeof deps.listNodePidsMatchingScript === 'function'
+      // Accept the singular seam name too, so a consumer that already holds the
+      // one-script helper can inject it unchanged.
+      ? ((scriptPaths, opts) => [...new Set(
+        (scriptPaths || []).flatMap(script => deps.listNodePidsMatchingScript(script, opts) || []),
+      )])
+      : shared.listNodePidsMatchingScripts);
+
+  const resolved = {
+    platform: deps.platform || process.platform,
+    now: typeof deps.now === 'function' ? deps.now : (() => Date.now()),
+    sleep: typeof deps.sleep === 'function' ? deps.sleep : (ms => shared.sleepMs(ms)),
+    statSync: typeof deps.statSync === 'function' ? deps.statSync : (p => fs.statSync(p)),
+    readFileSync: deps.readFileSync,
+    isPidAlive: typeof deps.isPidAlive === 'function' ? deps.isPidAlive : (pid => shared.isPidAlive(pid)),
+    listNodePidsMatchingScripts: listPlural,
+    killPidOnly: typeof deps.killPidOnly === 'function' ? deps.killPidOnly : defaultKillPidOnly,
+    writeStopIntent: typeof deps.writeStopIntent === 'function' ? deps.writeStopIntent : defaultWriteStopIntent,
+    requestEngineStop: typeof deps.requestEngineStop === 'function' ? deps.requestEngineStop : defaultRequestEngineStop,
+    waitForPidDeath: deps.waitForPidDeath,
+  };
+  if (typeof resolved.waitForPidDeath !== 'function') {
+    resolved.waitForPidDeath = (pid, timeoutMs) => defaultWaitForPidDeath(pid, timeoutMs, resolved);
+  }
+  return resolved;
+}
+
+/** Poll until `pid` is gone or the budget expires. Built entirely on the
+ *  injected liveness/clock/sleep seams so tests never sleep for real. */
+function defaultWaitForPidDeath(pid, timeoutMs, d) {
+  const start = d.now();
+  const n = Number(pid);
+  if (!Number.isInteger(n) || n <= 0) return { ok: true, waitedMs: 0 };
+  const budget = Math.max(0, Number(timeoutMs) || 0);
+  for (;;) {
+    let alive;
+    try { alive = d.isPidAlive(n) === true; } catch { alive = false; }
+    if (!alive) return { ok: true, waitedMs: d.now() - start };
+    const elapsed = d.now() - start;
+    if (elapsed >= budget) return { ok: false, waitedMs: elapsed };
+    try { d.sleep(Math.min(PID_POLL_MS, Math.max(1, budget - elapsed))); }
+    catch { return { ok: false, waitedMs: d.now() - start }; }
+  }
+}
+
+/**
+ * `-shm` evidence for the pinned root.
+ *
+ * SQLite creates the WAL shared-memory index when the FIRST connection opens
+ * the database and removes it when the LAST one closes cleanly, so a present,
+ * non-empty `-shm` means a process still holds the database open even when
+ * every PID we know about is gone.
+ *
+ * A missing file is genuine evidence of release; ANY other stat failure is a
+ * failed probe and must fail CLOSED, never be read as "released".
+ */
+function readShmEvidence(root, d) {
+  try {
+    const info = d.statSync(path.join(root, ...STATE_SHM_SEGMENTS));
+    return { probed: true, present: true, bytes: Number(info && info.size) || 0 };
+  } catch (err) {
+    const code = String((err && err.code) || '');
+    if (code === 'ENOENT' || code === 'ENOTDIR') return { probed: true, present: false, bytes: 0 };
+    return { probed: false, present: false, bytes: 0, error: String((err && err.message) || err) };
+  }
+}
+
+function shmHeld(evidence) {
+  return !!evidence.present && Number(evidence.bytes) > 0;
+}
+
+/**
+ * Does this holder mean a process of OURS is (or may still be) running?
+ *
+ * `unverified-identity` is the one reason that does NOT: the process table was
+ * readable and proved the recorded PID belongs to something else, i.e. a stale
+ * PID file. Callers that are about to spawn a REPLACEMENT stack use this to
+ * tell "a daemon survived, aborting is the only safe move" apart from "a PID
+ * file outlived its process, carry on" — the second is exactly the state
+ * `minions restart` exists to heal.
+ */
+function holderIsLiveDaemon(holder) {
+  if (!holder) return false;
+  return holder.reason !== STOP_STACK_HOLDER_REASONS.UNVERIFIED_IDENTITY;
+}
+
+/**
+ * Map a verdict to its process exit code. A refusal outranks a budget expiry:
+ * when identity could not be proven, nothing was signalled at all, so reporting
+ * "did not quiesce in time" would send the operator to widen a budget that was
+ * never the problem.
+ */
+function stopStackExitCode(verdict) {
+  if (verdict && verdict.ok === true) return STOP_STACK_EXIT.OK;
+  const holders = verdict && Array.isArray(verdict.holders) ? verdict.holders : [];
+  const probeFailures = verdict && Array.isArray(verdict.probeFailures) ? verdict.probeFailures : [];
+  const refused = holders.some(h => h && (
+    h.reason === STOP_STACK_HOLDER_REASONS.UNVERIFIED_IDENTITY
+    || h.reason === STOP_STACK_HOLDER_REASONS.IDENTITY_PROBE_FAILED));
+  if (refused || probeFailures.length > 0) return STOP_STACK_EXIT.REFUSED_UNVERIFIED;
+  return STOP_STACK_EXIT.NOT_QUIESCED;
+}
+
+/**
+ * @param {object}  opts
+ * @param {string}  opts.minionsHome    runtime root to stop; scopes every identity proof
+ * @param {number}  [opts.timeoutMs]    total budget for the whole sequence
+ * @param {string}  [opts.source]       recorded in stop-intent.json
+ * @param {boolean} [opts.waitForRelease=true]
+ *        Poll for `engine/state.db-shm` to be released, and require that
+ *        release for `ok`. Callers that immediately REPLACE the stack
+ *        (restart, the init upgrade) or DELETE the root (uninstall, nuke) pass
+ *        false: they still run the identical ordered teardown and still get the
+ *        `-shm` evidence reported, they simply do not block on it. Only a
+ *        pre-migration gate needs the guarantee.
+ * @param {object}  [opts.deps]         injectable process/clock seams
+ */
+function stopRuntimeStack({ minionsHome, timeoutMs, source, waitForRelease = true, deps } = {}) {
+  const d = resolveDeps(deps);
+  const steps = [];
+  const stopped = [];
+  const holders = new Map();
+  const probeFailures = [];
+  const startedAt = d.now();
+  const elapsed = () => Math.max(0, d.now() - startedAt);
+  const record = (step, detail) => { steps.push({ step, atMs: elapsed(), ...(detail || {}) }); };
+
+  if (!minionsHome || typeof minionsHome !== 'string') {
+    return {
+      ok: false,
+      stopped: [],
+      holders: [],
+      probeFailures: ['minionsHome must be a non-empty path string'],
+      shm: { probed: false, present: false, bytes: 0, released: false },
+      waitedMs: 0,
+      steps: [],
+      reason: 'refused: minionsHome must be a non-empty path string',
+    };
+  }
+
+  const root = path.resolve(minionsHome);
+  const scriptPaths = daemonScriptPathsForRoot(root);
+  const budget = Number.isFinite(Number(timeoutMs)) && Number(timeoutMs) > 0
+    ? Number(timeoutMs)
+    : DEFAULT_STOP_TIMEOUT_MS;
+  const deadline = startedAt + budget;
+  const remaining = () => Math.max(0, deadline - d.now());
+  /** Budget for a wait that follows a kill we ALREADY issued. Floored at
+   *  POST_KILL_GRACE_MS so a drain that consumed the whole budget cannot
+   *  collapse it to zero and turn a dying process into a reported holder. */
+  const postKillWaitMs = () => Math.max(remaining(), POST_KILL_GRACE_MS);
+
+  const services = readServiceRecords(root, d);
+  const byName = new Map(services.map(svc => [svc.name, svc]));
+  const byPid = new Map(services.filter(svc => svc.pid).map(svc => [svc.pid, svc]));
+
+  const isAlive = (pid) => {
+    try { return d.isPidAlive(pid) === true; }
+    catch (err) {
+      probeFailures.push(`liveness probe failed for PID ${pid}: ${String((err && err.message) || err)}`);
+      return false;
+    }
+  };
+  /** Owned PIDs for `paths`, or null when the process table could not be read. */
+  const ownedPids = (paths) => {
+    try {
+      const list = d.listNodePidsMatchingScripts(paths, { platform: d.platform }) || [];
+      return list.map(Number).filter(n => Number.isInteger(n) && n > 0);
+    } catch (err) {
+      probeFailures.push(`process identity probe failed: ${String((err && err.message) || err)}`);
+      return null;
+    }
+  };
+  const refuse = (pid, name, reason) => {
+    if (!holders.has(pid)) holders.set(pid, { name, pid, reason });
+  };
+  const kill = (pid) => {
+    try { return d.killPidOnly(pid, scriptPaths) !== false; }
+    catch { return false; }
+  };
+
+  // ── 1. stop-intent FIRST, so the supervisor stands down before anything dies
+  let intentOk = false;
+  try { intentOk = d.writeStopIntent(root, source || 'stopRuntimeStack') !== false; }
+  catch { intentOk = false; }
+  record(STOP_STACK_STEPS.STOP_INTENT, { ok: intentOk });
+
+  /** Stop one recorded service, refusing any live PID we cannot prove is ours. */
+  const stopVerifiedService = (svc, step) => {
+    if (!svc || !svc.pid) { record(step, { result: 'no-pid' }); return; }
+    if (svc.pid === process.pid) { record(step, { result: 'self', pid: svc.pid }); return; }
+    if (!isAlive(svc.pid)) { record(step, { result: 'already-stopped', pid: svc.pid }); return; }
+    const owned = ownedPids([svc.script]);
+    if (owned === null) {
+      refuse(svc.pid, svc.name, STOP_STACK_HOLDER_REASONS.IDENTITY_PROBE_FAILED);
+      record(step, { result: 'probe-failed', pid: svc.pid });
+      return;
+    }
+    if (!owned.includes(svc.pid)) {
+      refuse(svc.pid, svc.name, STOP_STACK_HOLDER_REASONS.UNVERIFIED_IDENTITY);
+      record(step, { result: 'refused-unverified', pid: svc.pid });
+      return;
+    }
+    const killed = kill(svc.pid);
+    if (killed) stopped.push({ name: svc.name, pid: svc.pid, via: 'process-kill' });
+    record(step, { result: killed ? 'stopped' : 'kill-failed', pid: svc.pid });
+  };
+
+  // ── 2. supervisor, ── 3. dashboard (the second holder of engine/state.db)
+  stopVerifiedService(byName.get('supervisor'), STOP_STACK_STEPS.SUPERVISOR);
+  stopVerifiedService(byName.get('dashboard'), STOP_STACK_STEPS.DASHBOARD);
+
+  // ── 4. ask the engine to drain gracefully — ALWAYS before any forced reap.
+  const engine = byName.get('engine');
+  const engineWasAlive = !!(engine && engine.pid) && isAlive(engine.pid);
+  let drain = { ok: false, error: 'not-requested' };
+  if (engineWasAlive) {
+    try {
+      drain = d.requestEngineStop({
+        minionsHome: root,
+        timeoutMs: Math.max(1, Math.min(ENGINE_STOP_CALL_TIMEOUT_MS, remaining() || ENGINE_STOP_CALL_TIMEOUT_MS)),
+      }) || { ok: false };
+    } catch (err) {
+      drain = { ok: false, error: String((err && err.message) || err) };
+    }
+  }
+  record(STOP_STACK_STEPS.ENGINE_DRAIN, {
+    requested: engineWasAlive,
+    ok: drain.ok === true,
+    ...(drain.error ? { error: drain.error } : {}),
+  });
+
+  // ── 5. give the drain its budget before escalating. Detached cold agents are
+  //       not part of that drain and stay alive for reattachment.
+  let drained = false;
+  if (engineWasAlive) {
+    const wait = d.waitForPidDeath(engine.pid, remaining()) || { ok: false };
+    drained = wait.ok === true;
+    if (drained) stopped.push({ name: 'engine', pid: engine.pid, via: 'graceful-drain' });
+    record(STOP_STACK_STEPS.ENGINE_WAIT, { pid: engine.pid, drained, waitedMs: wait.waitedMs });
+  } else {
+    record(STOP_STACK_STEPS.ENGINE_WAIT, { drained: false, result: 'not-running' });
+  }
+
+  // ── 6. identity-verified reap of whatever survived the drain.
+  const survivors = services.filter(svc => svc.pid && svc.pid !== process.pid && isAlive(svc.pid));
+  const reapOwned = survivors.length > 0 ? ownedPids(scriptPaths) : [];
+  const reaped = [];
+  for (const svc of survivors) {
+    if (reapOwned === null) { refuse(svc.pid, svc.name, STOP_STACK_HOLDER_REASONS.IDENTITY_PROBE_FAILED); continue; }
+    if (!reapOwned.includes(svc.pid)) { refuse(svc.pid, svc.name, STOP_STACK_HOLDER_REASONS.UNVERIFIED_IDENTITY); continue; }
+    if (!kill(svc.pid)) continue;
+    stopped.push({ name: svc.name, pid: svc.pid, via: 'process-kill' });
+    reaped.push(svc.pid);
+  }
+  for (const pid of reaped) d.waitForPidDeath(pid, postKillWaitMs());
+  record(STOP_STACK_STEPS.REAP, { candidates: survivors.map(s => s.pid), reaped });
+
+  // ── 7. wait for the WAL index to be released, unless the caller opted out
+  //       because it is about to replace the stack or delete the root.
+  let shm = readShmEvidence(root, d);
+  let shmPolls = 1;
+  while (waitForRelease && shm.probed && shmHeld(shm) && remaining() > 0) {
+    try { d.sleep(Math.min(SHM_POLL_MS, remaining())); } catch { break; }
+    shm = readShmEvidence(root, d);
+    shmPolls++;
+  }
+  record(STOP_STACK_STEPS.SHM_WAIT, {
+    polls: shmPolls, present: shm.present, bytes: shm.bytes, awaited: waitForRelease,
+  });
+
+  // ── 8. ONE late-respawn sweep. The watchdog (or a surviving supervisor) can
+  //       spawn a fresh engine/dashboard inside the kill window; those become
+  //       orphans that re-create -shm right after the probe went green. Sweep
+  //       BEFORE the final verification so the verdict cannot be stale.
+  //       stop-intent is still set, so any respawn has already stood down.
+  const lateOwned = ownedPids(scriptPaths);
+  const swept = [];
+  for (const pid of (lateOwned || [])) {
+    if (pid === process.pid) continue;
+    if (!kill(pid)) continue;
+    const known = byPid.get(pid);
+    stopped.push({ name: known ? known.name : 'daemon', pid, via: 'late-sweep' });
+    swept.push(pid);
+  }
+  for (const pid of swept) d.waitForPidDeath(pid, postKillWaitMs());
+  record(STOP_STACK_STEPS.LATE_SWEEP, { swept });
+
+  // ── 9. final verification, strictly after the sweep.
+  for (const svc of services) {
+    if (!svc.pid || svc.pid === process.pid) continue;
+    if (isAlive(svc.pid)) {
+      refuse(svc.pid, svc.name,
+        holders.has(svc.pid) ? holders.get(svc.pid).reason : STOP_STACK_HOLDER_REASONS.STILL_ALIVE);
+    }
+  }
+  shm = readShmEvidence(root, d);
+  const heldShm = shmHeld(shm);
+  record(STOP_STACK_STEPS.VERIFY, {
+    holders: holders.size,
+    shmPresent: shm.present,
+    shmBytes: shm.bytes,
+    shmProbed: shm.probed,
+  });
+
+  const holderList = [...holders.values()];
+  const releaseVerified = shm.probed && !heldShm;
+  const shmReport = {
+    probed: shm.probed,
+    present: shm.present,
+    bytes: shm.bytes,
+    released: releaseVerified,
+    ...(shm.error ? { error: shm.error } : {}),
+  };
+  const ok = holderList.length === 0
+    && probeFailures.length === 0
+    && (waitForRelease ? releaseVerified : true);
+  let reason;
+  if (holderList.length > 0) {
+    reason = `still holding ${root}: ${holderList.map(h => `${h.name} (PID ${h.pid}) [${h.reason}]`).join(', ')}`;
+  } else if (probeFailures.length > 0) {
+    reason = `could not verify the runtime stack: ${probeFailures.join('; ')}`;
+  } else if (!waitForRelease && !releaseVerified) {
+    reason = 'runtime stack stopped; the engine/state.db-shm release was not awaited '
+      + `(${shm.probed ? `${shm.bytes} bytes still present` : `probe failed: ${shm.error}`})`;
+  } else if (!shm.probed) {
+    reason = `could not read engine/state.db-shm: ${shm.error}`;
+  } else if (heldShm) {
+    reason = `engine/state.db-shm is still present and non-empty (${shm.bytes} bytes), `
+      + 'so a process still holds the database open';
+  } else {
+    reason = 'runtime stack stopped; no process holds engine/state.db';
+  }
+
+  return { ok, stopped, holders: holderList, probeFailures, shm: shmReport, waitedMs: elapsed(), steps, reason };
+}
+
+module.exports = {
+  stopRuntimeStack,
+  stopStackExitCode,
+  holderIsLiveDaemon,
+  daemonScriptPathsForRoot,
+  readServiceRecords,
+  defaultRequestEngineStop,
+  STOP_STACK_STEPS,
+  STOP_STACK_HOLDER_REASONS,
+  STOP_STACK_EXIT,
+  DEFAULT_STOP_TIMEOUT_MS,
+  POST_KILL_GRACE_MS,
+  PID_POLL_MS,
+  SHM_POLL_MS,
+  ENGINE_STOP_CALL_TIMEOUT_MS,
+};