omp-conductor 0.15.3 → 0.15.5

package/src/lifecycle.ts

@@ -364,9 +364,19 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
  * Same ownership rule as {@link stopDaemon}: when the unit owns the live pid,
  * `systemctl restart` is the *only* path — a failed manager call is terminal,
  * never a fallthrough to raw signals. An *unanswered* ownership query is also
- * terminal: "dbus blipped" is not "no unit". Falling back to stop+start is
- * reserved for a confirmed hand-started daemon (no systemd, inactive unit, or
- * a unit whose MainPID is someone else).
+ * terminal: "dbus blipped" is not "no unit". A *failed* installed unit is the
+ * third non-fallthrough: it must be restored through the manager
+ * (`reset-failed` + `start`), because a detached CLI daemon beside a failed
+ * unit is exactly the "restarted, but the fleet is still down" lie this
+ * module exists to prevent (#376). Falling back to stop+start is reserved for
+ * a confirmed hand-started daemon (no systemd, inactive unit, or a unit whose
+ * MainPID is someone else).
+ *
+ * Success is reported only after the service manager is re-probed and proven
+ * to own the daemon: the unit's MainPID must equal the daemon record pid AND
+ * `/healthz` must answer. The manager call returning is not that proof — for
+ * `Type=simple` the unit is active as soon as the process is forked, before
+ * the daemon has bound its port.
  *
  * Returns the record of the process that is now answering `/healthz`.
  */
@@ -379,6 +389,16 @@ export async function restartDaemon(
     throw new Error(ownershipUnknown("restart", ownership.reason));
   }
 
+  if (ownership.kind === "failed") {
+    // An installed unit in failed state: reset the failure record and let
+    // systemd start the service. Never fall through to stopDaemon() +
+    // startDaemon() — that leaves the unit failed while handing the operator
+    // a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
+    // Any manager refusal is terminal; so is unproven ownership afterwards.
+    const record = await restoreFailedUnit(o.timeoutMs);
+    return { previous, record, via: "systemctl" };
+  }
+
   const unitOwns =
     ownership.kind === "active" && (previous === undefined || previous.pid === ownership.pid);
 
@@ -393,26 +413,95 @@ export async function restartDaemon(
     // systemctl restart returns once the new MainPID is up; the pidfile is
     // written by the daemon itself on boot, so wait for that rather than
     // inventing a record from the unit alone.
-    const deadline = Date.now() + READY_TIMEOUT_MS;
-    for (;;) {
-      const rec = livingDaemon();
-      if (rec !== undefined) {
-        const health = await healthCheck(rec.port);
-        if (health.ok) return { previous, record: rec, via: "systemctl" };
-      }
-      if (Date.now() >= deadline) break;
-      await sleep(READY_POLL_MS);
-    }
-    throw new Error(
-      `systemctl restart ${SYSTEMD_UNIT} returned, but the daemon never answered /healthz`,
-    );
+    const record = await waitForOwnedDaemon("restart", o.timeoutMs);
+    return { previous, record, via: "systemctl" };
   }
 
+  // Confirmed unmanaged: no unit, an inactive unit, or a unit whose MainPID
+  // is somebody else. The detached CLI daemon is the only path left.
   await stopDaemon({ timeoutMs: o.timeoutMs });
   const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
   return { previous, record, via: "cli" };
 }
 
+/**
+ * Restores a failed installed unit through the manager: `reset-failed` then
+ * `start`, and proves the result the same way any managed restart is proven
+ * (see {@link waitForOwnedDaemon}). Refusal or unproven ownership is terminal
+ * — never a fallthrough to the detached CLI daemon.
+ */
+async function restoreFailedUnit(timeoutMs?: number): Promise<DaemonRecord> {
+  const reset = systemctl(["reset-failed", SYSTEMD_UNIT]);
+  if (!reset.ok) {
+    throw new Error(systemctlFailure("reset-failed", reset));
+  }
+  const started = systemctl(["start", SYSTEMD_UNIT]);
+  if (!started.ok) {
+    throw new Error(systemctlFailure("start", started));
+  }
+  return await waitForOwnedDaemon("start", timeoutMs);
+}
+
+/**
+ * Waits for the daemon a manager call was just made to boot, and refuses to
+ * report success until the service manager is proven to own it: the unit's
+ * MainPID must equal the daemon record pid AND `/healthz` must answer. A unit
+ * that went back to `failed` is a confirmed negative and fails fast — waiting
+ * cannot un-fail it. Any other unproven state fails at the deadline with a
+ * diagnostic that names the mismatch, never a bare "not ready".
+ */
+async function waitForOwnedDaemon(verb: "restart" | "start", timeoutMs?: number): Promise<DaemonRecord> {
+  const via = `systemctl ${verb} ${SYSTEMD_UNIT} returned`;
+  const inspect = `systemctl status ${SYSTEMD_UNIT}`;
+  const deadline = Date.now() + (timeoutMs ?? READY_TIMEOUT_MS);
+  for (;;) {
+    const rec = livingDaemon();
+    const ownership = probeUnit();
+    if (rec !== undefined && ownership.kind === "active" && ownership.pid === rec.pid) {
+      const health = await healthCheck(rec.port);
+      if (health.ok) return rec;
+    } else if (ownership.kind === "failed") {
+      // A confirmed negative: the start did not take and the unit is failed
+      // again. Waiting longer cannot un-fail it.
+      throw new Error(
+        `${via}, but the unit went back to failed — the daemon is NOT running; check \`${inspect}\``,
+      );
+    }
+    if (Date.now() >= deadline) break;
+    await sleep(READY_POLL_MS);
+  }
+
+  // Failed to prove ownership. Name what is wrong rather than a bare timeout,
+  // so the operator knows whether to fix the unit or hunt the foreign pid.
+  const rec = livingDaemon();
+  const ownership = probeUnit();
+  if (rec === undefined) {
+    throw new Error(
+      `${via}, but no live daemon record appeared — the daemon did not come back; check \`${inspect}\``,
+    );
+  }
+  if (ownership.kind === "active") {
+    if (ownership.pid === rec.pid) {
+      throw new Error(
+        `${via}, but the daemon never answered /healthz on :${rec.port} — ` +
+          `the service manager owns pid ${rec.pid}, but it is not serving; ` +
+          `check \`${inspect}\` and ${rec.logFile}`,
+      );
+    }
+    throw new Error(
+      `${via}, but the unit's MainPID (${ownership.pid}) does not match the daemon record pid (${rec.pid}) — ` +
+        `the reported pid is NOT owned by the service manager; check \`${inspect}\``,
+    );
+  }
+  if (ownership.kind === "unknown") {
+    throw new Error(
+      `${via}, but ownership can no longer be determined (${ownership.reason}) — ` +
+        `cannot confirm the daemon is managed; check \`${inspect}\``,
+    );
+  }
+  throw new Error(`${via}, but the unit is not active; check \`${inspect}\``);
+}
+
 /**
  * What we know about {@link SYSTEMD_UNIT}.
  *
@@ -420,14 +509,22 @@ export async function restartDaemon(
  *   states (`activating`, `deactivating`, `reloading`, `reactivating`) — the
  *   pid is still systemd-owned, so a raw SIGTERM would bounce under
  *   `Restart=on-failure`.
- * - `inactive` — confirmed not running (MainPID absent/0), unit absent, *or*
- *   no `systemctl` binary on this host. Safe to treat as "not supervised here".
+ * - `failed` — the unit is installed and its last activation failed
+ *   (ActiveState=failed, no MainPID). systemd still owns this unit's
+ *   lifecycle, so restoring the daemon must go through the manager
+ *   (`reset-failed` + `start`); classifying a failed unit as "not supervised
+ *   here" is how `restart` launched an unmanaged daemon next to a unit that
+ *   stayed failed (#376).
+ * - `inactive` — confirmed not running (MainPID absent/0, ActiveState not
+ *   `failed`), unit absent, *or* no `systemctl` binary on this host. Safe to
+ *   treat as "not supervised here".
  * - `unknown` — the manager exists (or we cannot tell it does not) but the
  *   query failed: dbus blip, permission, timeout. Must not be collapsed into
  *   `inactive` — that is how a unit-owned daemon gets a raw SIGTERM.
  */
 export type UnitOwnership =
   | { kind: "active"; pid: number }
+  | { kind: "failed" }
   | { kind: "inactive" }
   | { kind: "unknown"; reason: string };
 
@@ -454,26 +551,39 @@ export function probeUnit(unit = SYSTEMD_UNIT): UnitOwnership {
   // live pid (> 1) means systemd still owns that process — including during
   // `activating` / `deactivating` / `reloading`. Filtering on ActiveState here
   // used to label those transitional states "inactive" and hand the pid to a
-  // raw SIGTERM, which is exactly the Restart=on-failure bounce.
+  // raw SIGTERM, which is exactly the Restart=on-failure bounce. The one
+  // place ActiveState decides is the `failed` classification below: with no
+  // MainPID there is no pid to hand to anything, but the unit has not gone
+  // away either.
   const lines = ran.stdout
     .split("\n")
     .map((l) => l.trim())
     .filter((l) => l.length > 0);
   let pid: number | undefined;
+  let activeState: string | undefined;
   for (const line of lines) {
     if (/^\d+$/.test(line)) {
       const n = Number(line);
       if (Number.isInteger(n) && n > 1) pid = n;
+    } else {
+      activeState = line;
     }
   }
-  if (pid === undefined) return { kind: "inactive" };
-  return { kind: "active", pid };
+  if (pid !== undefined) return { kind: "active", pid };
+  // A failed unit is NOT the confirmed negative `inactive` claims to be: the
+  // unit is installed and the manager still owns its lifecycle, so restoring
+  // the daemon has to go through systemd (`reset-failed` + `start`). Reading
+  // `failed` as "not supervised here" is precisely how #376 ended with a
+  // healthy-looking but unmanaged pid next to a unit that stayed failed.
+  if (activeState === "failed") return { kind: "failed" };
+  return { kind: "inactive" };
 }
 
 /**
  * The MainPID of an *active* {@link SYSTEMD_UNIT}, or `undefined` when the
- * unit is confirmed inactive/absent or when ownership could not be determined.
- * Prefer {@link probeUnit} when the caller must distinguish those two.
+ * unit is confirmed inactive, failed, or absent, or when ownership could not
+ * be determined. Prefer {@link probeUnit} when the caller must distinguish
+ * those two.
  */
 export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
   const ownership = probeUnit(unit);
@@ -490,8 +600,9 @@ export function systemdMainPid(unit = SYSTEMD_UNIT): number | undefined {
  * - `stop` — unit is active and owns `pid` (or there is no pidfile and the
  *   unit is the only candidate). Caller MUST go through systemctl; a failed
  *   manager call is terminal.
- * - `not-ours` — confirmed inactive/absent unit, no systemd binary, or a unit
- *   whose MainPID is someone else. Caller may SIGTERM its own pidfile process.
+ * - `not-ours` — confirmed inactive/failed/absent unit, no systemd binary, or
+ *   a unit whose MainPID is someone else. Caller may SIGTERM its own pidfile
+ *   process (a failed unit owns nothing, so signalling its record is safe).
  * - `unknown` — ownership query failed. Caller MUST NOT signal.
  */
 type SystemdStopDecision =
@@ -502,7 +613,10 @@ type SystemdStopDecision =
 function decideSystemdStop(pid: number | undefined): SystemdStopDecision {
   const ownership = probeUnit();
   if (ownership.kind === "unknown") return { kind: "unknown", reason: ownership.reason };
-  if (ownership.kind === "inactive") return { kind: "not-ours" };
+  // A failed unit owns no process (MainPID 0 — systemd has already reaped or
+  // lost it), so a live record pid is a confirmed "not ours", exactly like an
+  // inactive unit. Restoring the unit is restart's job, not stop's.
+  if (ownership.kind === "inactive" || ownership.kind === "failed") return { kind: "not-ours" };
   if (pid !== undefined && ownership.pid !== pid) return { kind: "not-ours" };
   return { kind: "stop", mainPid: ownership.pid };
 }
@@ -533,11 +647,17 @@ async function runSystemdStop(pid: number, timeoutMs?: number): Promise<void> {
   }
 }
 
-function systemctlFailure(verb: "stop" | "restart", ran: SystemctlResult): string {
+function systemctlFailure(
+  verb: "stop" | "restart" | "reset-failed" | "start",
+  ran: SystemctlResult,
+): string {
   const detail = (ran.stderr.trim() || ran.stdout.trim() || "no output").split("\n")[0] ?? "no output";
+  const guard =
+    verb === "stop" || verb === "restart"
+      ? `refusing to signal a unit-owned daemon (that is how Restart=on-failure turns stop into a bounce)`
+      : `the unit was NOT restored, so the daemon would be unmanaged`;
   return (
-    `systemctl ${verb} ${SYSTEMD_UNIT} failed: ${detail} — ` +
-    `refusing to signal a unit-owned daemon (that is how Restart=on-failure turns stop into a bounce); ` +
+    `systemctl ${verb} ${SYSTEMD_UNIT} failed: ${detail} — ${guard}; ` +
     `fix the unit or run \`systemctl ${verb} ${SYSTEMD_UNIT}\` yourself`
   );
 }

package/src/omp.ts

@@ -398,6 +398,15 @@ const DRAIN_GRACE_MS = 2_000;
 /** How long a disposed child gets to exit before it is signalled. */
 const DISPOSE_GRACE_MS = 5_000;
 
+/**
+ * Thrown by {@link createSession} when the pre-spawn admission gate closes: a
+ * daemon stop landed while the session socket was binding, so the child was
+ * never spawned. The caller maps this to a stopped run, not a failed one —
+ * a shutdown must not charge an attempt against a worker it refused to start
+ * (#374).
+ */
+export class SessionAdmissionClosedError extends Error {}
+
 export interface CreateSessionOptions {
   cwd: string;
   sessionDir?: string;
@@ -430,6 +439,16 @@ export interface CreateSessionOptions {
    * its verb channel to this pid, and a channel accepts nothing until it is bound.
    */
   onSpawn?: (pid: number) => void;
+  /**
+   * Pre-spawn admission gate (#374). Consulted once, immediately after the
+   * socket bind await and immediately before `Bun.spawn` — the last window a
+   * daemon stop can land in before a child exists. A gate that returns false
+   * closes the listener, removes the socket, and throws
+   * {@link SessionAdmissionClosedError} instead of spawning; the run then
+   * settles as stopped rather than adding a worker the shutdown would have to
+   * wait for. Absent, the spawn always proceeds.
+   */
+  maySpawn?: () => boolean;
   /** Child stderr, line by line. Defaults to the process's own stderr. */
   onChildLog?: (line: string) => void;
   startupTimeoutMs?: number;
@@ -505,6 +524,21 @@ export async function createSession(opts: CreateSessionOptions): Promise<AgentSe
       resolve();
     });
   });
+  // Pre-spawn admission recheck (#374): SIGTERM/SIGINT can land during the
+  // listen await above — after the daemon's own pre-launch checks and before
+  // this child exists. A closed gate means no child: close the listener,
+  // remove the socket (and, when this side created the directory, the
+  // directory itself — the same cleanup the failure paths below run), and
+  // signal the caller to settle the run as stopped instead of spawning a
+  // worker the shutdown would then have to wait for.
+  if (opts.maySpawn?.() === false) {
+    server.close();
+    rmSync(socketPath, { force: true });
+    if (owned) rmSync(socketDir, { recursive: true, force: true });
+    throw new SessionAdmissionClosedError(
+      "daemon shutdown began while the session socket was binding",
+    );
+  }
   // The socket is the run's own channel, and the daemon's own uid is the only
   // one that speaks on it.
   chmodSync(socketPath, 0o600);

package/src/types.ts

@@ -1154,6 +1154,7 @@ export type AdmissionHoldReason =
   | "unsalvaged-wip"
   | "daily-spend-cap"
   | "plan-usage-cap"
+  | "shutting-down"
   | "unroutable:no-repo-label"
   | "unroutable:multiple-repo-labels"
   | "unroutable:unknown-repo";