@lifeaitools/clauth 2.15.2 → 2.15.3

package/cli/ops/pm2-adapter.js

@@ -53,6 +53,116 @@ function callbackCall(pm2, method, args = []) {
   });
 }
 
+// PM2's own stop/restart/delete callbacks resolve once the daemon has ISSUED
+// the kill, not once the OS has actually reclaimed the process -- confirmed
+// live 2026-09-04, Windows: a `restart` on a real app left its outgoing
+// child alive and still holding its listen port for 1000+ cycles, and a
+// plain `stop` right before an `npm install -g` over that same app's
+// install directory still hit EBUSY because the "stopped" process was, in
+// fact, not. Every caller of this adapter (the ops CLI, ops HTTP surface,
+// and every cutover procedure built on top of it) inherits that lie unless
+// this file closes it once, here, rather than each caller re-discovering it
+// under pressure.
+function isAlive(pid) {
+  if (typeof pid !== "number" || pid <= 0) return false;
+  try {
+    process.kill(pid, 0);
+    return true;
+  } catch (error) {
+    // ESRCH: no such process -- genuinely gone. Anything else (EPERM, etc.)
+    // means the OS still has an opinion about this pid, so it isn't safe to
+    // call it dead.
+    return error.code !== "ESRCH";
+  }
+}
+
+// pm2's own death check on Windows is also parent-pid-only (God.processIsDead
+// reads pm2_env._tree_pids || [pid], and treekill's taskkill callback reports
+// only the pid it was given) -- so a spawned-child topology like clauth's own
+// next-runner fixture can have its child survive everything pm2 itself
+// thinks killed it. Capture _tree_pids alongside pid so this adapter isn't
+// blind to exactly the shape its own fixtures use.
+function pidsOf(processes) {
+  return (Array.isArray(processes) ? processes : [processes])
+    .flatMap((entry) => [entry?.pid, ...(entry?.pm2_env?._tree_pids || [])])
+    .filter((pid) => typeof pid === "number" && pid > 0);
+}
+
+const DEATH_TIMEOUT_MS = 3000;
+const KILL_TIMEOUT_MS = 1000;
+
+// describe() resolving with [] IS the normal "no such app" case -- do not
+// mask that behind a try/catch that also swallows a real RPC/transport
+// failure. A caller that can't see whether the old process is still alive
+// must not silently report success; let it fail loudly instead.
+function pidsFor(pm2, target) {
+  return callbackCall(pm2, "describe", [target]).then(pidsOf);
+}
+
+async function waitForDeath(pids, { pollMs = 150, timeoutMs = DEATH_TIMEOUT_MS } = {}) {
+  const deadline = Date.now() + timeoutMs;
+  let survivors = pids.filter(isAlive);
+  while (survivors.length > 0 && Date.now() < deadline) {
+    await new Promise((resolve) => setTimeout(resolve, pollMs));
+    survivors = survivors.filter(isAlive);
+  }
+  return survivors;
+}
+
+// Verifies the pids captured BEFORE a stop/restart/reload/delete are
+// actually gone afterward, escalating to SIGKILL (TerminateProcess on
+// Windows, same as POSIX SIGKILL) if PM2's own signal didn't finish the job
+// within the timeout. Only throws if a process survives an explicit SIGKILL,
+// which should not happen -- if it does, the caller needs to know its
+// "stop" silently didn't, rather than proceed as though it did. The thrown
+// error carries the surviving pids as structured fields (not just in the
+// message) so a caller logging/job-recording the failure can act on them
+// even after this operation's own bookkeeping (e.g. a deleted pm2 record)
+// is gone.
+async function verifyTerminated(target, oldPids, operation) {
+  if (oldPids.length === 0) return;
+  const survivors = await waitForDeath(oldPids);
+  if (survivors.length === 0) return;
+  console.error(
+    `[ops] pm2 ${operation} "${target}" did not stop pid(s) ${survivors.join(", ")} within ${DEATH_TIMEOUT_MS}ms -- force-killing`,
+  );
+  for (const pid of survivors) {
+    try {
+      process.kill(pid, "SIGKILL");
+    } catch {
+      // Already gone between the check and the kill -- fine.
+    }
+  }
+  const stillAlive = await waitForDeath(survivors, { timeoutMs: KILL_TIMEOUT_MS });
+  if (stillAlive.length > 0) {
+    const error = new Error(
+      `pm2 ${operation} "${target}": pid(s) ${stillAlive.join(", ")} survived SIGKILL -- refusing to report success`,
+    );
+    error.pids = stillAlive;
+    error.operation = operation;
+    error.target = target;
+    error.reason = "survived_sigkill";
+    throw error;
+  }
+}
+
+// restart/reload spawn the replacement BEFORE the pm2 callback this adapter
+// awaits resolves (confirmed against pm2's own God.restartProcessId, and
+// reload falls through to the identical path outside cluster_mode -- which
+// is every app this control plane manages). So by the time verification
+// starts, a genuinely new process can already exist, and on Windows a freed
+// pid can be reallocated to it inside the same poll window this adapter is
+// watching. Re-describing AFTER the operation and only verifying death for
+// pids no longer reported as current tells the two apart: a truly orphaned
+// old pid is never in the fresh describe and still gets killed; a pid pm2
+// still reports as live is never touched, whether that's because pm2
+// genuinely didn't rotate it or because it now belongs to the new process.
+async function survivorsOf(pm2, target, oldPids) {
+  if (oldPids.length === 0) return [];
+  const currentPids = new Set(await pidsFor(pm2, target));
+  return oldPids.filter((pid) => !currentPids.has(pid));
+}
+
 function normalizeProcess(process) {
   if (!process || typeof process !== "object") return process;
   return {
@@ -101,10 +211,43 @@ export function createPm2Adapter(pm2) {
         case "logs": return (await callbackCall(pm2, "describe", [missing(input.target, "target")])).map(normalizeProcess);
         case "bus": return callbackCall(pm2, "launchBus");
         case "start": return callbackCall(pm2, "start", [missing(input.script, "script"), input.options || {}]);
-        case "stop": return callbackCall(pm2, "stop", [missing(input.target, "target")]);
-        case "restart": return callbackCall(pm2, "restart", [missing(input.target, "target"), input.options || {}]);
-        case "reload": return callbackCall(pm2, "reload", [missing(input.target, "target"), input.options || {}]);
-        case "delete": return callbackCall(pm2, "delete", [missing(input.target, "target")]);
+        case "stop": {
+          const target = missing(input.target, "target");
+          const oldPids = await pidsFor(pm2, target);
+          const result = await callbackCall(pm2, "stop", [target]);
+          await verifyTerminated(target, oldPids, "stop");
+          return result;
+        }
+        case "restart": {
+          const target = missing(input.target, "target");
+          const oldPids = await pidsFor(pm2, target);
+          const result = await callbackCall(pm2, "restart", [target, input.options || {}]);
+          await verifyTerminated(target, await survivorsOf(pm2, target, oldPids), "restart");
+          return result;
+        }
+        case "reload": {
+          // Fork mode -- every app this control plane manages -- makes
+          // reload identical to restart inside pm2 (God.reloadProcessId only
+          // takes the cluster branch in cluster_mode; everything else falls
+          // straight through to God.restartProcessId). clauth's own deploy
+          // cutover (deployment-adapter.js) calls this operation, not
+          // "restart" -- so this is the actual path "install then hit
+          // restart, pm2 should pick up the new one" travels in production,
+          // and it needs the identical guarantee, not a lighter one because
+          // the pm2 verb sounds gentler.
+          const target = missing(input.target, "target");
+          const oldPids = await pidsFor(pm2, target);
+          const result = await callbackCall(pm2, "reload", [target, input.options || {}]);
+          await verifyTerminated(target, await survivorsOf(pm2, target, oldPids), "reload");
+          return result;
+        }
+        case "delete": {
+          const target = missing(input.target, "target");
+          const oldPids = await pidsFor(pm2, target);
+          const result = await callbackCall(pm2, "delete", [target]);
+          await verifyTerminated(target, oldPids, "delete");
+          return result;
+        }
         case "scale": return callbackCall(pm2, "scale", [missing(input.target, "target"), missing(input.instances, "instances")]);
         case "reset": return callbackCall(pm2, "reset", [missing(input.target, "target")]);
         case "dump": return callbackCall(pm2, "dump");

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@lifeaitools/clauth",
-  "version": "2.15.2",
+  "version": "2.15.3",
   "description": "Hardware-bound credential vault for the LIFEAI infrastructure stack",
   "type": "module",
   "bin": {