omp-conductor 0.18.1 → 0.19.0

package/src/setup-install.ts

@@ -20,7 +20,8 @@ import { platform } from "node:os";
 import { join } from "node:path";
 import { loadConfig, stateDir } from "./config.ts";
 import { pauseInstance, pauseSourceToken, setPaused, statusSnapshot } from "./daemon.ts";
-import { fleetLayers, DEFAULT_HERDR_UNIT } from "./fleet.ts";
+import { fleetLayers, DEFAULT_HERDR_SESSION, DEFAULT_HERDR_UNIT } from "./fleet.ts";
+import { spawnSync } from "node:child_process";
 import { livingDaemon } from "./lifecycle.ts";
 import { dbPath, openStore } from "./store.ts";
 import {
@@ -51,8 +52,9 @@ import { runPrivileged, type PrivilegedDeps, type PrivilegedStep } from "./privi
 import {
   agentRenameVerdict,
   checkEscalation,
+  checkSessionRestartContext,
+  herdrOwnership,
   DEFAULT_AGENT_RENAME_DEPS,
-  defaultIdentityProbes,
   defaultServiceRuntime,
   planHostRuntime,
   totalConfiguredWorkers,
@@ -62,8 +64,9 @@ import {
   RECOVER_SCRIPT_INSTALL_PATH,
   type AgentRenameDeps,
   type EscalationDeps,
+  type HerdrOwnership,
+  type HerdrOwnershipFacts,
   type ServiceRuntime,
-  type WorkerIdentityProbes,
 } from "./setup-host.ts";
 import type { WizardUi } from "./wizard-ui.ts";
 import type { Caps, ProjectConfig, Store } from "./types.ts";
@@ -84,14 +87,6 @@ export interface InstallDeps {
   escalation?: EscalationDeps;
   /** `"linux"` gates the systemd half. Injectable so the refusal is testable. */
   platform?: () => string;
-  /**
-   * The worker identity plan's probes (#798). `undefined` means the real
-   * host's read-only facts — production: every `setup host` installs and
-   * grants the dedicated worker account. `null` disables the identity plan
-   * entirely (the historical step list, for tests pinning it), and an object
-   * is used as given (tests pinning a specific host state).
-   */
-  workerIdentityProbes?: WorkerIdentityProbes | null;
   unitDir?: string;
   /**
    * Where the recovery playbook installs. Injectable so the idempotency gate
@@ -133,6 +128,31 @@ export interface InstallDeps {
    * default is the same `herdr --session` CLI recover.sh and the tick use.
    */
   agentRename?: AgentRenameDeps;
+  /**
+   * The environment the caller's context is read from (#834): `$HERDR_ENV` says
+   * this shell is a pane inside the unit the install restarts. Injectable so
+   * the refusal is testable without a live Herdr session.
+   */
+  env?: Record<string, string | undefined>;
+  /**
+   * The live supervision facts and the one remedy for them (#893). Injectable
+   * so the ownership transfer and its readiness gate are testable without
+   * touching real systemd or a real herdr server — production reads
+   * `herdr session list --json` and `systemctl show`.
+   */
+  supervision?: SupervisionDeps;
+}
+
+/** The reads and the one write the #893 ownership transfer needs. */
+export interface SupervisionDeps {
+  /** Who is serving the fleet session right now. */
+  facts: (session: string) => HerdrOwnershipFacts;
+  /** Bounded readiness polling between reads. */
+  sleep: (ms: number) => Promise<void>;
+  /** How many times the post-restart gate re-reads before giving up. */
+  attempts?: number;
+  /** Milliseconds between those reads. */
+  intervalMs?: number;
 }
 
 /**
@@ -199,13 +219,6 @@ export async function runHostInstall(
   // happens only after consent (#510), so the prompt's "Nothing has been run
   // yet" is true of the staged tree when it is printed, and a declined run
   // leaves every staged file byte-identical.
-  // The worker identity plan rides the same consent (#798): production probes
-  // the real host (account present? grants current?); a test may pass its own
-  // probes, or `null` to plan the historical step list only.
-  const identityProbes: WorkerIdentityProbes | null =
-    deps.workerIdentityProbes === null
-      ? null
-      : deps.workerIdentityProbes ?? defaultIdentityProbes(deps.runtime ?? defaultServiceRuntime(telegramStateDir));
   const plan = planHostRuntime(
     project,
     caps,
@@ -215,7 +228,6 @@ export async function runHostInstall(
     unitDir,
     deps.recoverScriptInstallPath ?? RECOVER_SCRIPT_INSTALL_PATH,
     deps.multiProject ?? hostMultiProject(),
-    identityProbes,
   );
   // No pane-shell key is planned (unusable login shell, unparseable config):
   // the operator hears why before the consent prompt, not after an install
@@ -257,6 +269,17 @@ export async function runHostInstall(
     return { kind: "installed", wrote: [] };
   }
 
+  // The caller must survive the transaction it started (#834). Placed here on
+  // purpose: after the plan and the no-op gate — so a host with no herdr unit,
+  // and a re-run with nothing to install, are never refused for a restart that
+  // is not going to happen — and before the drain fence and the post-consent
+  // staging, so a refusal has mutated nothing at all.
+  const context = checkSessionRestartContext("setup host", plan.steps, deps.env);
+  if (context.kind === "refuse") {
+    ui.notify(context.message, "error");
+    return { kind: "refused", reason: context.message };
+  }
+
   // argv, not shell: `installCommands` renders `sudo …` strings for humans to
   // read, and re-parsing those into an argv is how a path with a space becomes
   // two arguments. The plan carries the exact argv once, in
@@ -298,7 +321,47 @@ export async function runHostInstall(
     attributeRestartKills(fence.scope, project?.name ?? "", deps.drain?.drainStore);
   };
 
-  const outcome = await runPrivileged(plan.steps, ui, {
+  // Who is serving the fleet session, before anything is staged or run (#893).
+  // A `systemctl restart` that returns 0 does not prove supervision moved: an
+  // unmanaged `herdr --session <name> server` holding the session makes every
+  // start of the unit exit 1, and `Type=simple` returns before the child finds
+  // out. So the collision is resolved by stopping the unmanaged session inside
+  // this transaction, or the transaction refuses before it mutates anything.
+  const supervision = deps.supervision ?? defaultSupervisionDeps();
+  const session = plan.herdrSession ?? DEFAULT_HERDR_SESSION;
+  let transferSteps: PrivilegedStep[] = [];
+  if (plan.herdrUnit !== undefined) {
+    const ownership = herdrOwnership(supervision.facts(session));
+    if (ownership.kind === "unknown") {
+      // The remedy here stops somebody's live session, so an unreadable fact is
+      // a refusal, never an assumption. Nothing has been staged or installed.
+      const reason =
+        `refused: ${ownership.detail}, so this install cannot tell whether ${DEFAULT_HERDR_UNIT} owns the ` +
+        `fleet session or an unmanaged server does. Stopping the wrong one kills a live fleet, so nothing ` +
+        `was staged or installed. Check \`systemctl status ${DEFAULT_HERDR_UNIT}\` and ` +
+        `\`herdr session list\`, then re-run.`;
+      ui.notify(reason, "error");
+      return { kind: "refused", reason };
+    }
+    if (ownership.kind === "unmanaged") {
+      ui.notify(
+        `The fleet session is served outside systemd: ${ownership.detail}. This install stops that server ` +
+          `first so ${DEFAULT_HERDR_UNIT} can take the session over — every pane in it restarts under the unit.`,
+        "warning",
+      );
+      transferSteps = [
+        {
+          // The fleet account's own session, so no escalation: `setup host`
+          // already runs as that account (the escalation guard proves it).
+          title: `stop the unmanaged herdr session "${session}" so ${DEFAULT_HERDR_UNIT} can own it`,
+          argv: ["herdr", "session", "stop", session],
+          unprivileged: true,
+        },
+      ];
+    }
+  }
+
+  const outcome = await runPrivileged([...transferSteps, ...plan.steps], ui, {
     ...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
     title: "Install and start the supervised session?",
     answerKey: "install-host",
@@ -317,14 +380,22 @@ export async function runHostInstall(
                 ]),
           ]),
       "The unit runs as the account that staged it; nothing here changes that.",
-      ...(plan.workerIdentity === undefined
+      ...(transferSteps.length === 0
+        ? []
+        : [
+            `Stops the unmanaged herdr session "${session}" first — systemd cannot adopt a running process, so the ` +
+              "session has to be handed over. Every pane in it comes back under the unit.",
+          ]),
+      // Retirement is named before the confirm that authorises it (#895): an
+      // operator approving this batch is approving an unmount, and a removal
+      // nobody announced is not consented to.
+      ...(plan.retire === undefined
         ? []
         : [
-            `Establishes the dedicated worker identity ${plan.workerIdentity.account}: every worker session runs under it, ` +
-              "never under the daemon's account, and it is granted search/read access to exactly the fleet paths a worker needs.",
-            ...(plan.workerIdentity.current
-              ? ["The worker identity is already in place — nothing to change."]
-              : plan.workerIdentity.pending),
+            `Retires host state an earlier release installed and this one no longer ships: ` +
+              `${[...plan.retire.units, ...plan.retire.staged].join(", ")}.`,
+            "Worker sessions run as the fleet account again, so the read-only bind that served the separate " +
+              "worker account is unmounted and its unit removed. The account itself is left alone.",
           ]),
       "",
       liveWorkers === 0
@@ -343,6 +414,27 @@ export async function runHostInstall(
     if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
     return { kind: "failed", reason: `${outcome.step.title} exited ${outcome.exitCode}` };
   }
+  // Success is not "the steps exited 0" (#893). `Type=simple` returns before the
+  // session server can discover a collision and die, so the transaction proves
+  // the canonical unit is still up AND owns the session — bounded, because an
+  // auto-restart loop keeps looking momentarily alive.
+  if (plan.herdrUnit !== undefined) {
+    const settled = await proveSupervision(supervision, session);
+    if (settled.kind !== "unit") {
+      // Dispatch stays paused, deliberately and unlike a failed step: the fleet
+      // is being served by something nobody supervises, and admitting workers
+      // into that is the outcome this gate exists to prevent. The operator is
+      // told what was observed rather than a generic failure.
+      const reason =
+        `${DEFAULT_HERDR_UNIT} did not take over the fleet session: ${settled.detail}. ` +
+        `The units are installed, but supervision did not transfer, so dispatch stays paused. ` +
+        `Check \`systemctl status ${DEFAULT_HERDR_UNIT}\` and \`herdr session list\`, then re-run \`setup host\` ` +
+        `from a shell outside the session.`;
+      ui.notify(reason, "error");
+      return { kind: "failed", reason };
+    }
+    ui.notify(`${DEFAULT_HERDR_UNIT} owns the fleet session — ${settled.detail}.`, "info");
+  }
   if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
   ui.notify(`Installed and started ${STAGED_SERVICE_NAME}.`, "info");
   // #541: the tick plan may have restamped agentName away from the shared
@@ -376,6 +468,84 @@ export async function runHostInstall(
   return { kind: "installed", wrote };
 }
 
+/**
+ * Poll until the canonical unit owns the fleet session, or the bound runs out
+ * (#893). Returns the last verdict either way, so the caller reports what was
+ * actually observed rather than "timed out".
+ *
+ * `absent` and `unmanaged` are both failures here, for different reasons: the
+ * first is a unit that started and died (the collision's own signature under
+ * `Type=simple`), the second is a session still held outside it.
+ */
+async function proveSupervision(deps: SupervisionDeps, session: string): Promise<HerdrOwnership> {
+  const attempts = deps.attempts ?? SUPERVISION_PROBE_ATTEMPTS;
+  const intervalMs = deps.intervalMs ?? SUPERVISION_PROBE_INTERVAL_MS;
+  let verdict = herdrOwnership(deps.facts(session));
+  for (let attempt = 1; attempt < attempts && verdict.kind !== "unit"; attempt += 1) {
+    await deps.sleep(intervalMs);
+    verdict = herdrOwnership(deps.facts(session));
+  }
+  return verdict;
+}
+
+/** How many reads the readiness gate takes, and how far apart. Twelve seconds
+ *  total: long enough for a session server to bind its socket, short enough
+ *  that a restart loop is reported rather than waited out. */
+const SUPERVISION_PROBE_ATTEMPTS = 12;
+const SUPERVISION_PROBE_INTERVAL_MS = 1_000;
+
+/** The real reads: herdr's own session list, and systemd's view of the unit. */
+function defaultSupervisionDeps(): SupervisionDeps {
+  return {
+    facts: (session) => ({
+      sessionRunning: herdrSessionRunning(session),
+      unit: herdrUnitState(),
+    }),
+    sleep: Bun.sleep,
+  };
+}
+
+/** Whether herdr reports a live server for one session, or `undefined` when it
+ *  could not be asked — never `false` on a failed read (#893). */
+function herdrSessionRunning(session: string): boolean | undefined {
+  const ran = spawnSync("herdr", ["session", "list", "--json"], { encoding: "utf8", timeout: 10_000 });
+  if (ran.status !== 0 || typeof ran.stdout !== "string") return undefined;
+  try {
+    const parsed: unknown = JSON.parse(ran.stdout);
+    const sessions = parsed !== null && typeof parsed === "object" ? Reflect.get(parsed, "sessions") : undefined;
+    if (!Array.isArray(sessions)) return undefined;
+    const row = sessions.find(
+      (entry) => entry !== null && typeof entry === "object" && Reflect.get(entry, "name") === session,
+    );
+    if (row === undefined) return false;
+    return Reflect.get(row as object, "running") === true;
+  } catch {
+    return undefined;
+  }
+}
+
+/** The canonical unit's live state, or `undefined` when systemctl could not
+ *  answer. A property systemd omits reads as unreadable rather than as zero. */
+function herdrUnitState(): HerdrOwnershipFacts["unit"] {
+  const ran = spawnSync(
+    "systemctl",
+    ["show", DEFAULT_HERDR_UNIT, "-p", "ActiveState", "-p", "SubState", "-p", "MainPID", "-p", "NRestarts"],
+    { encoding: "utf8", timeout: 10_000 },
+  );
+  if (ran.status !== 0 || typeof ran.stdout !== "string") return undefined;
+  const values = new Map<string, string>();
+  for (const line of ran.stdout.split("\n")) {
+    const eq = line.indexOf("=");
+    if (eq > 0) values.set(line.slice(0, eq), line.slice(eq + 1));
+  }
+  const activeState = values.get("ActiveState");
+  const subState = values.get("SubState");
+  const mainPid = Number(values.get("MainPID"));
+  const nRestarts = Number(values.get("NRestarts"));
+  if (activeState === undefined || subState === undefined || !Number.isFinite(mainPid)) return undefined;
+  return { activeState, subState, mainPid, nRestarts: Number.isFinite(nRestarts) ? nRestarts : 0 };
+}
+
 /**
  * The pause/drain fence's default accessors, overridable per test through
  * {@link InstallDeps.drain.deps}. Mirrors `upgrade.ts`'s {@link DrainDeps}
@@ -395,7 +565,14 @@ function hostInstallDrainDeps(ui: WizardUi, deps: InstallDeps): DrainDeps {
         : { running: true, project: daemon.project, generation: `${daemon.pid}@${daemon.startedAt}` };
     },
     pauseState: (project) => pauseInstance(project),
-    setPaused: (v, project) => setPaused(v, { source: pauseSourceToken("setup host"), reason: "setup host, draining" }, project),
+    setPaused: (v, project) =>
+      // `owner` is this process (#938): the fence lives exactly as long as this
+      // transaction, so a kill leaves it behind with nothing to explain it.
+      setPaused(
+        v,
+        { source: pauseSourceToken("setup host"), reason: "setup host, draining", owner: process.pid },
+        project,
+      ),
     sleep: Bun.sleep,
     log: (message) => ui.notify(message, "info"),
   };