omp-conductor 0.15.11 → 0.15.13

package/src/setup-install.ts

@@ -18,7 +18,17 @@
 import { existsSync, readFileSync } from "node:fs";
 import { platform } from "node:os";
 import { join } from "node:path";
-import { stateDir } from "./config.ts";
+import { loadConfig, stateDir } from "./config.ts";
+import { pauseInstance, pauseSourceToken, setPaused, statusSnapshot } from "./daemon.ts";
+import { fleetLayers, DEFAULT_HERDR_UNIT } from "./fleet.ts";
+import { livingDaemon } from "./lifecycle.ts";
+import { dbPath, openStore } from "./store.ts";
+import {
+  resolveScope,
+  pauseAndDrain,
+  type DrainDeps,
+  type UpgradeScope,
+} from "./upgrade.ts";
 import {
   graphRepos,
   formatGraphSetup,
@@ -34,18 +44,21 @@ import {
 import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
 import { runPrivileged, type PrivilegedDeps, type PrivilegedStep } from "./privileged.ts";
 import {
+  agentRenameVerdict,
   checkEscalation,
+  DEFAULT_AGENT_RENAME_DEPS,
   planHostRuntime,
   totalConfiguredWorkers,
   writeHostRuntime,
   STAGED_SERVICE_NAME,
   SYSTEMD_UNIT_DIR,
+  RECOVER_SCRIPT_INSTALL_PATH,
+  type AgentRenameDeps,
   type EscalationDeps,
   type ServiceRuntime,
 } from "./setup-host.ts";
-import { DEFAULT_HERDR_UNIT } from "./fleet.ts";
 import type { WizardUi } from "./wizard-ui.ts";
-import type { Caps, ProjectConfig } from "./types.ts";
+import type { Caps, ProjectConfig, Store } from "./types.ts";
 
 /**
  * Every outcome a caller has to tell apart. `staged` is the non-Linux answer:
@@ -64,12 +77,46 @@ export interface InstallDeps {
   /** `"linux"` gates the systemd half. Injectable so the refusal is testable. */
   platform?: () => string;
   unitDir?: string;
+  /**
+   * Where the recovery playbook installs. Injectable so the idempotency gate
+   * is testable hermetically; the real install uses
+   * {@link RECOVER_SCRIPT_INSTALL_PATH}.
+   */
+  recoverScriptInstallPath?: string;
   /**
    * The runtime the plan derives from. Injectable so tests pin herdr
    * present/absent deterministically; the default ({@link defaultServiceRuntime})
    * detects herdr from the host's own PATH.
    */
   runtime?: ServiceRuntime;
+  /**
+   * Whether the host configures more than one project, which scopes the
+   * recovery unit's RECOVER_PROJECT (host-global: never one project's name on
+   * a multi-project host, #510/#530). Injectable so tests pin the shape
+   * hermetically; the default ({@link hostMultiProject}) reads the config.
+   */
+  multiProject?: boolean;
+  /**
+   * The pause/drain fence around the restart (#512): which projects to drain,
+   * how long to wait, and the store used to attribute a run the restart kills.
+   * Injectable so tests pin the drain hermetically; the defaults read the real
+   * daemon, sentinel and store.
+   */
+  drain?: {
+    deps?: DrainDeps;
+    /** How long the drain waits for live workers before giving up and staying
+     *  paused; absent waits indefinitely, the `upgrade` posture. */
+    timeoutMs?: number;
+    /** The store consulted for runs still live at restart time, so they are
+     *  attributed before the session host restarts under them. */
+    drainStore?: Pick<Store, "liveRuns" | "updateRun">;
+  };
+  /**
+   * The live herdr CLI seam for the post-install agent rename (#541).
+   * Injectable so tests pin the verdict without touching the host's herdr; the
+   * default is the same `herdr --session` CLI recover.sh and the tick use.
+   */
+  agentRename?: AgentRenameDeps;
 }
 
 /**
@@ -100,11 +147,21 @@ function sameFile(a: string, b: string): boolean {
 }
 
 /**
- * Install the supervised daemon unit: re-render, stage, then run the four steps
- * `planHostRuntime` used to only print.
+ * Install the supervised daemon unit: re-render, stage, then run the plan's
+ * steps in order. The steps are the plan's own {@link HostRuntimePlan.steps} —
+ * the same single list its {@link installCommands} display renders from — so
+ * what is printed and what is executed can never diverge.
  */
 export async function runHostInstall(
-  project: ProjectConfig,
+  /**
+   * The project whose per-project tail to write, or `undefined` for a
+   * host-global install: the units are host-global (same staged paths, same
+   * systemd destinations, one shared daemon), so no project still installs
+   * and starts them. The per-project tail (tick config, brief link) is the
+   * only part that needs a name — with none, it is skipped with a printed
+   * note (#530).
+   */
+  project: ProjectConfig | undefined,
   caps: Caps,
   telegramStateDir: string,
   ui: WizardUi,
@@ -116,40 +173,106 @@ export async function runHostInstall(
     return { kind: "refused", reason: verdict.message };
   }
 
-  const plan = planHostRuntime(project, caps, telegramStateDir, deps.runtime, hostInstallWorkers(caps));
-  const { wrote, warnings } = writeHostRuntime(plan);
-  for (const warning of warnings) ui.notify(warning, "warning");
   const unitDir = deps.unitDir ?? SYSTEMD_UNIT_DIR;
+  // The plan is computed eagerly — `planHostRuntime` renders every byte and
+  // compares it against disk, so the prompt can print exactly what would be
+  // written and the currentInstall gate below can decide a re-run restarts
+  // nothing. But rendering is read-only: no file is written here. Staging
+  // happens only after consent (#510), so the prompt's "Nothing has been run
+  // yet" is true of the staged tree when it is printed, and a declined run
+  // leaves every staged file byte-identical.
+  const plan = planHostRuntime(
+    project,
+    caps,
+    telegramStateDir,
+    deps.runtime,
+    hostInstallWorkers(caps),
+    unitDir,
+    deps.recoverScriptInstallPath ?? RECOVER_SCRIPT_INSTALL_PATH,
+    deps.multiProject ?? hostMultiProject(),
+  );
+  // No pane-shell key is planned (unusable login shell, unparseable config):
+  // the operator hears why before the consent prompt, not after an install
+  // that quietly had nothing to say about the shell.
+  if (plan.herdrConfigWarning !== undefined) ui.notify(plan.herdrConfigWarning, "warning");
+  if (plan.herdrConfigProblem !== undefined) ui.notify(plan.herdrConfigProblem, "warning");
+  // A host-global install (no --project): the per-project tail was
+  // deliberately not written, and the operator must be told which files and
+  // how to get them — never a silent skip.
+  if (plan.noProject !== undefined) {
+    ui.notify(
+      `No project named: ${plan.noProject.skipped.join(", ")} were not written. ${plan.noProject.how}.`,
+      "info",
+    );
+  }
   const installed = join(unitDir, STAGED_SERVICE_NAME);
   const installedHerdr = join(unitDir, DEFAULT_HERDR_UNIT);
 
   const blocked = linuxOnly(deps);
   if (blocked !== undefined) {
+    // A non-Linux host cannot run systemd, so staging IS the deliverable: the
+    // rendered files are written for the operator to copy to the box that will
+    // run them, and only the systemctl half is refused. There is no consent
+    // prompt on this path — nothing privileged to approve — so the write lands
+    // here, not in the (never reached) post-consent hook.
+    const { wrote, warnings } = writeHostRuntime(plan);
+    for (const warning of warnings) ui.notify(warning, "warning");
     ui.notify(`${blocked} ${plan.service.path}`, "warning");
     return { kind: "staged", wrote, reason: `${blocked} ${plan.service.path}` };
   }
 
+  // A no-op re-run: every unit the steps would install is already current on
+  // disk, so installing them again would only touch mtimes and restart the
+  // units for nothing. Report success without running a single privileged step.
+  // Nothing is staged here either: a re-run with nothing to install has no
+  // reason to touch the staged tree.
+  if (plan.currentInstall) {
+    ui.notify("The supervised daemon and recovery unit are already current — nothing to install, nothing restarted.", "info");
+    return { kind: "installed", wrote: [] };
+  }
+
   // argv, not shell: `installCommands` renders `sudo …` strings for humans to
   // read, and re-parsing those into an argv is how a path with a space becomes
-  // two arguments. The steps are built from the same values instead.
-  const steps: PrivilegedStep[] = [
-    { title: `install ${STAGED_SERVICE_NAME}`, argv: ["install", "-m", "0644", plan.service.path, installed] },
-    { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
-    { title: `enable ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "enable", STAGED_SERVICE_NAME] },
-    { title: `restart ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "restart", STAGED_SERVICE_NAME] },
-    // The herdr session server, provisioned alongside the daemon (#456). Only
-    // when herdr is installed and the plan therefore staged a unit.
-    ...(plan.herdrUnit === undefined
-      ? []
-      : [
-          { title: `install ${DEFAULT_HERDR_UNIT}`, argv: ["install", "-m", "0644", plan.herdrUnit.path, installedHerdr] },
-          { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
-          { title: `enable ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "enable", DEFAULT_HERDR_UNIT] },
-          { title: `restart ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "restart", DEFAULT_HERDR_UNIT] },
-        ]),
-  ];
-
-  const outcome = await runPrivileged(steps, ui, {
+  // two arguments. The plan carries the exact argv once, in
+  // {@link HostRuntimePlan.steps}; we execute that same list, so the printed
+  // plan and the run are the same steps (#509).
+  //
+  // The fence `upgrade` already has (#512): pause new issue claims, drain live
+  // workers to idle, restart the units, then restore the dispatch state this
+  // began in. The consent prompt below names how many workers will be drained
+  // so the operator can choose the moment; it authorises the whole batch,
+  // drain included, in one confirm (the `beforeRun` hook).
+  const drainDeps = hostInstallDrainDeps(ui, deps);
+  // Host-wide by default (#389): the daemon this restarts serves every
+  // configured project, so the drain covers all of them — narrowing to the
+  // named project would restart the shared session host under the other
+  // projects' live workers without ever counting them.
+  const fenceScope = resolveScope(drainDeps, "setup host");
+  const liveWorkers = fenceScope.selectors.reduce((n, s) => n + drainDeps.snapshot(s).liveWorkers, 0);
+  let fence: { scope: UpgradeScope; initialPaused: boolean } | undefined;
+  // What the post-consent staging actually wrote, surfaced by the outcomes that
+  // report it. Empty on a decline — the hook never ran.
+  let wrote: readonly string[] = [];
+  const beforeRun = async (): Promise<void> => {
+    // Consent has been given; the plan's files may now land. Staging here —
+    // post-confirm, pre-step, under the same confirm that authorised the batch
+    // — is what makes the prompt's "Nothing has been run yet" true of the
+    // staged tree when it is printed: a declined prompt never reaches this
+    // hook, so nothing is staged and a re-run still plans an install (#510).
+    const staged = writeHostRuntime(plan);
+    wrote = staged.wrote;
+    for (const warning of staged.warnings) ui.notify(warning, "warning");
+    fence = await pauseAndDrain(drainDeps, "setup host", {
+      ...(deps.drain?.timeoutMs === undefined ? {} : { timeoutMs: deps.drain.timeoutMs }),
+    });
+    // Any run still live when the session host is about to restart is killed by
+    // it; record the actor on its row so the classifier reads an administrative
+    // kill rather than charging a failure for an operator's restart (#512). The
+    // fence drained to idle, so in the healthy path this records nothing.
+    attributeRestartKills(fence.scope, project?.name ?? "", deps.drain?.drainStore);
+  };
+
+  const outcome = await runPrivileged(plan.steps, ui, {
     ...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
     title: "Install and start the supervised session?",
     preamble: [
@@ -159,18 +282,126 @@ export async function runHostInstall(
         : [
             `Also installs the fleet session unit as ${installedHerdr} with the account's login shell so panes`,
             "run it instead of dash, then enables and restarts it.",
+            ...(plan.herdrConfigTarget === undefined
+              ? []
+              : [
+                  `And merges [terminal] default_shell into ${plan.herdrConfigTarget} as you, before that restart,`,
+                  "so the session server reads the pinned shell the moment it comes up.",
+                ]),
           ]),
       "The unit runs as the account that staged it; nothing here changes that.",
+      "",
+      liveWorkers === 0
+        ? "No workers are live — nothing to drain before the restart."
+        : `Pauses new claims and drains ${liveWorkers} live worker(s) to idle first — this can take minutes per worker — then restarts the units and resumes dispatch.`,
     ],
+    beforeRun,
   });
   if (outcome.kind === "declined") return { kind: "declined", wrote };
   if (outcome.kind === "failed") {
+    // Restore the preceding dispatch state: a restart that failed is still a
+    // finished operator action whose units may or may not be up, so the fleet
+    // must not sit paused for it. A throw from the drain itself (a timeout or
+    // a superseded fence) never reaches here — it propagates and leaves
+    // dispatch paused, fail-closed, exactly as upgrade does.
+    if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
     return { kind: "failed", reason: `${outcome.step.title} exited ${outcome.exitCode}` };
   }
+  if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
   ui.notify(`Installed and started ${STAGED_SERVICE_NAME}.`, "info");
+  // #541: the tick plan may have restamped agentName away from the shared
+  // default — the exact moment the live pane is stuck under the old name and
+  // every tick declines with the rename hint. Perform the rename now that the
+  // herdr session has (re)started, safely: exactly one pane carrying the old
+  // identity is renamed; zero, several, or an unreadable agent list prints the
+  // exact command and says why it was not run. Never guesses a pane.
+  if (plan.agentRename !== undefined) {
+    const renameDeps = deps.agentRename ?? DEFAULT_AGENT_RENAME_DEPS;
+    const verdict = agentRenameVerdict(
+      renameDeps.list(plan.agentRename.session),
+      plan.agentRename.from,
+      plan.agentRename.to,
+      plan.agentRename.session,
+    );
+    if (verdict.kind === "rename") {
+      const ran = renameDeps.rename(plan.agentRename.session, verdict.pane, plan.agentRename.to);
+      if (ran.ok) {
+        ui.notify(`Renamed the herdr agent in pane ${verdict.pane} to ${plan.agentRename.to}.`, "info");
+      } else {
+        ui.notify(
+          `The herdr agent rename did not go through (${ran.problem ?? "unknown"}): ${verdict.command} — the pane still carries the old name.`,
+          "warning",
+        );
+      }
+    } else {
+      ui.notify(`${verdict.reason}; run it by hand: ${verdict.command}`, "warning");
+    }
+  }
   return { kind: "installed", wrote };
 }
 
+/**
+ * The pause/drain fence's default accessors, overridable per test through
+ * {@link InstallDeps.drain.deps}. Mirrors `upgrade.ts`'s {@link DrainDeps}
+ * wiring, except the pause names `setup host` as the actor and the drain
+ * reports progress through the same UI surface.
+ */
+function hostInstallDrainDeps(ui: WizardUi, deps: InstallDeps): DrainDeps {
+  if (deps.drain?.deps !== undefined) return deps.drain.deps;
+  return {
+    snapshot: statusSnapshot,
+    layers: fleetLayers,
+    projectNames: () => loadConfig().projects.map((p) => p.name),
+    daemonIdentity: () => {
+      const daemon = livingDaemon();
+      return daemon === undefined
+        ? { running: false }
+        : { running: true, project: daemon.project, generation: `${daemon.pid}@${daemon.startedAt}` };
+    },
+    pauseState: (project) => pauseInstance(project),
+    setPaused: (v, project) => setPaused(v, { source: pauseSourceToken("setup host"), reason: "setup host, draining" }, project),
+    sleep: Bun.sleep,
+    log: (message) => ui.notify(message, "info"),
+  };
+}
+
+/**
+ * Attribute the runs the restart is about to kill: any run still live when the
+ * session host restarts underneath it is that restart's casualty, not a failed
+ * attempt, so its row records the actor. The classifier reads this as an
+ * administrative kill that costs no budget and requeues (#512). The fence
+ * drains to idle first, so in the healthy path this records nothing — it is
+ * the safety net for a claim that raced the drain or a worker the drain could
+ * not bring to idle.
+ */
+function attributeRestartKills(
+  scope: UpgradeScope,
+  project: string,
+  store: Pick<Store, "liveRuns" | "updateRun"> | undefined,
+): void {
+  const attribute = (owner: Pick<Store, "liveRuns" | "updateRun">): void => {
+    for (const selector of scope.selectors) {
+      for (const run of owner.liveRuns(selector ?? project)) {
+        owner.updateRun(run.id, { lastError: "admin restart: setup host (draining)" });
+      }
+    }
+  };
+  if (store !== undefined) {
+    attribute(store);
+    return;
+  }
+  const path = dbPath();
+  // No store yet — a host that has never dispatched — means nothing to attribute
+  // and nothing a fresh open would do but create an empty database.
+  if (!existsSync(path)) return;
+  const owner = openStore(path);
+  try {
+    attribute(owner);
+  } finally {
+    owner.close();
+  }
+}
+
 /** Fleet-wide worker sum when config is loadable; otherwise this project's caps. */
 function hostInstallWorkers(caps: Caps): number {
   try {
@@ -180,6 +411,22 @@ function hostInstallWorkers(caps: Caps): number {
   }
 }
 
+/**
+ * Whether this host configures more than one project. The recovery unit is
+ * host-global and must not encode one project's name when several are
+ * configured, so the plan scopes RECOVER_PROJECT by this (#510/#530). A
+ * loadable config is the truth; when it cannot be read (a not-yet-configured
+ * box), there is no evidence of several projects and the named case stays
+ * unambiguous — the same fallback shape as {@link hostInstallWorkers}.
+ */
+function hostMultiProject(): boolean {
+  try {
+    return loadConfig().projects.length > 1;
+  } catch {
+    return false;
+  }
+}
+
 export interface GraphInstallOptions extends InstallDeps {
   /**
    * Skip the seeding step. The timer is still installed and enabled, and the