omp-conductor 0.15.10 → 0.15.12

package/src/setup-install.ts

@@ -15,10 +15,20 @@
  * call in here; `privileged.ts` stays the primitive underneath.
  */
 
-import { existsSync } from "node:fs";
+import { existsSync, readFileSync } from "node:fs";
 import { platform } from "node:os";
 import { join } from "node:path";
-import { stateDir } from "./config.ts";
+import { loadConfig, stateDir } from "./config.ts";
+import { pauseInstance, setPaused, statusSnapshot } from "./daemon.ts";
+import { fleetLayers, DEFAULT_HERDR_UNIT } from "./fleet.ts";
+import { livingDaemon } from "./lifecycle.ts";
+import { dbPath, openStore } from "./store.ts";
+import {
+  resolveScope,
+  pauseAndDrain,
+  type DrainDeps,
+  type UpgradeScope,
+} from "./upgrade.ts";
 import {
   graphRepos,
   formatGraphSetup,
@@ -40,10 +50,12 @@ import {
   writeHostRuntime,
   STAGED_SERVICE_NAME,
   SYSTEMD_UNIT_DIR,
+  RECOVER_SCRIPT_INSTALL_PATH,
   type EscalationDeps,
+  type ServiceRuntime,
 } from "./setup-host.ts";
 import type { WizardUi } from "./wizard-ui.ts";
-import type { Caps, ProjectConfig } from "./types.ts";
+import type { Caps, ProjectConfig, Store } from "./types.ts";
 
 /**
  * Every outcome a caller has to tell apart. `staged` is the non-Linux answer:
@@ -62,6 +74,40 @@ export interface InstallDeps {
   /** `"linux"` gates the systemd half. Injectable so the refusal is testable. */
   platform?: () => string;
   unitDir?: string;
+  /**
+   * Where the recovery playbook installs. Injectable so the idempotency gate
+   * is testable hermetically; the real install uses
+   * {@link RECOVER_SCRIPT_INSTALL_PATH}.
+   */
+  recoverScriptInstallPath?: string;
+  /**
+   * The runtime the plan derives from. Injectable so tests pin herdr
+   * present/absent deterministically; the default ({@link defaultServiceRuntime})
+   * detects herdr from the host's own PATH.
+   */
+  runtime?: ServiceRuntime;
+  /**
+   * Whether the host configures more than one project, which scopes the
+   * recovery unit's RECOVER_PROJECT (host-global: never one project's name on
+   * a multi-project host, #510/#530). Injectable so tests pin the shape
+   * hermetically; the default ({@link hostMultiProject}) reads the config.
+   */
+  multiProject?: boolean;
+  /**
+   * The pause/drain fence around the restart (#512): which projects to drain,
+   * how long to wait, and the store used to attribute a run the restart kills.
+   * Injectable so tests pin the drain hermetically; the defaults read the real
+   * daemon, sentinel and store.
+   */
+  drain?: {
+    deps?: DrainDeps;
+    /** How long the drain waits for live workers before giving up and staying
+     *  paused; absent waits indefinitely, the `upgrade` posture. */
+    timeoutMs?: number;
+    /** The store consulted for runs still live at restart time, so they are
+     *  attributed before the session host restarts under them. */
+    drainStore?: Pick<Store, "liveRuns" | "updateRun">;
+  };
 }
 
 /**
@@ -76,11 +122,37 @@ function linuxOnly(deps: InstallDeps): string | undefined {
 }
 
 /**
- * Install the supervised daemon unit: re-render, stage, then run the four steps
- * `planHostRuntime` used to only print.
+ * Whether two files hold identical bytes. The "keep" side of setup-host.ts's
+ * `actionFor`: a unit systemd already reads that matches what would be written
+ * needs no reinstall. Reads can fail (permissions, vanished mid-flight) — a
+ * read error is a mismatch, never silently "matches", because the whole point
+ * is to make the plan only claim a unit install is redundant when it can see
+ * the installed copy.
+ */
+function sameFile(a: string, b: string): boolean {
+  try {
+    return readFileSync(a, "utf8") === readFileSync(b, "utf8");
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Install the supervised daemon unit: re-render, stage, then run the plan's
+ * steps in order. The steps are the plan's own {@link HostRuntimePlan.steps} —
+ * the same single list its {@link installCommands} display renders from — so
+ * what is printed and what is executed can never diverge.
  */
 export async function runHostInstall(
-  project: ProjectConfig,
+  /**
+   * The project whose per-project tail to write, or `undefined` for a
+   * host-global install: the units are host-global (same staged paths, same
+   * systemd destinations, one shared daemon), so no project still installs
+   * and starts them. The per-project tail (tick config, brief link) is the
+   * only part that needs a name — with none, it is skipped with a printed
+   * note (#530).
+   */
+  project: ProjectConfig | undefined,
   caps: Caps,
   telegramStateDir: string,
   ui: WizardUi,
@@ -92,43 +164,206 @@ export async function runHostInstall(
     return { kind: "refused", reason: verdict.message };
   }
 
-  const plan = planHostRuntime(project, caps, telegramStateDir, undefined, hostInstallWorkers(caps));
-  const wrote = writeHostRuntime(plan);
   const unitDir = deps.unitDir ?? SYSTEMD_UNIT_DIR;
+  // The plan is computed eagerly — `planHostRuntime` renders every byte and
+  // compares it against disk, so the prompt can print exactly what would be
+  // written and the currentInstall gate below can decide a re-run restarts
+  // nothing. But rendering is read-only: no file is written here. Staging
+  // happens only after consent (#510), so the prompt's "Nothing has been run
+  // yet" is true of the staged tree when it is printed, and a declined run
+  // leaves every staged file byte-identical.
+  const plan = planHostRuntime(
+    project,
+    caps,
+    telegramStateDir,
+    deps.runtime,
+    hostInstallWorkers(caps),
+    unitDir,
+    deps.recoverScriptInstallPath ?? RECOVER_SCRIPT_INSTALL_PATH,
+    deps.multiProject ?? hostMultiProject(),
+  );
+  // No pane-shell key is planned (unusable login shell, unparseable config):
+  // the operator hears why before the consent prompt, not after an install
+  // that quietly had nothing to say about the shell.
+  if (plan.herdrConfigProblem !== undefined) ui.notify(plan.herdrConfigProblem, "warning");
+  // A host-global install (no --project): the per-project tail was
+  // deliberately not written, and the operator must be told which files and
+  // how to get them — never a silent skip.
+  if (plan.noProject !== undefined) {
+    ui.notify(
+      `No project named: ${plan.noProject.skipped.join(", ")} were not written. ${plan.noProject.how}.`,
+      "info",
+    );
+  }
   const installed = join(unitDir, STAGED_SERVICE_NAME);
+  const installedHerdr = join(unitDir, DEFAULT_HERDR_UNIT);
 
   const blocked = linuxOnly(deps);
   if (blocked !== undefined) {
+    // A non-Linux host cannot run systemd, so staging IS the deliverable: the
+    // rendered files are written for the operator to copy to the box that will
+    // run them, and only the systemctl half is refused. There is no consent
+    // prompt on this path — nothing privileged to approve — so the write lands
+    // here, not in the (never reached) post-consent hook.
+    const { wrote, warnings } = writeHostRuntime(plan);
+    for (const warning of warnings) ui.notify(warning, "warning");
     ui.notify(`${blocked} ${plan.service.path}`, "warning");
     return { kind: "staged", wrote, reason: `${blocked} ${plan.service.path}` };
   }
 
+  // A no-op re-run: every unit the steps would install is already current on
+  // disk, so installing them again would only touch mtimes and restart the
+  // units for nothing. Report success without running a single privileged step.
+  // Nothing is staged here either: a re-run with nothing to install has no
+  // reason to touch the staged tree.
+  if (plan.currentInstall) {
+    ui.notify("The supervised daemon and recovery unit are already current — nothing to install, nothing restarted.", "info");
+    return { kind: "installed", wrote: [] };
+  }
+
   // argv, not shell: `installCommands` renders `sudo …` strings for humans to
   // read, and re-parsing those into an argv is how a path with a space becomes
-  // two arguments. The steps are built from the same values instead.
-  const steps: PrivilegedStep[] = [
-    { title: `install ${STAGED_SERVICE_NAME}`, argv: ["install", "-m", "0644", plan.service.path, installed] },
-    { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
-    { title: `enable ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "enable", STAGED_SERVICE_NAME] },
-    { title: `restart ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "restart", STAGED_SERVICE_NAME] },
-  ];
-
-  const outcome = await runPrivileged(steps, ui, {
+  // two arguments. The plan carries the exact argv once, in
+  // {@link HostRuntimePlan.steps}; we execute that same list, so the printed
+  // plan and the run are the same steps (#509).
+  //
+  // The fence `upgrade` already has (#512): pause new issue claims, drain live
+  // workers to idle, restart the units, then restore the dispatch state this
+  // began in. The consent prompt below names how many workers will be drained
+  // so the operator can choose the moment; it authorises the whole batch,
+  // drain included, in one confirm (the `beforeRun` hook).
+  const drainDeps = hostInstallDrainDeps(ui, deps);
+  // Host-wide by default (#389): the daemon this restarts serves every
+  // configured project, so the drain covers all of them — narrowing to the
+  // named project would restart the shared session host under the other
+  // projects' live workers without ever counting them.
+  const fenceScope = resolveScope(drainDeps, "setup host");
+  const liveWorkers = fenceScope.selectors.reduce((n, s) => n + drainDeps.snapshot(s).liveWorkers, 0);
+  let fence: { scope: UpgradeScope; initialPaused: boolean } | undefined;
+  // What the post-consent staging actually wrote, surfaced by the outcomes that
+  // report it. Empty on a decline — the hook never ran.
+  let wrote: readonly string[] = [];
+  const beforeRun = async (): Promise<void> => {
+    // Consent has been given; the plan's files may now land. Staging here —
+    // post-confirm, pre-step, under the same confirm that authorised the batch
+    // — is what makes the prompt's "Nothing has been run yet" true of the
+    // staged tree when it is printed: a declined prompt never reaches this
+    // hook, so nothing is staged and a re-run still plans an install (#510).
+    const staged = writeHostRuntime(plan);
+    wrote = staged.wrote;
+    for (const warning of staged.warnings) ui.notify(warning, "warning");
+    fence = await pauseAndDrain(drainDeps, "setup host", {
+      ...(deps.drain?.timeoutMs === undefined ? {} : { timeoutMs: deps.drain.timeoutMs }),
+    });
+    // Any run still live when the session host is about to restart is killed by
+    // it; record the actor on its row so the classifier reads an administrative
+    // kill rather than charging a failure for an operator's restart (#512). The
+    // fence drained to idle, so in the healthy path this records nothing.
+    attributeRestartKills(fence.scope, project?.name ?? "", deps.drain?.drainStore);
+  };
+
+  const outcome = await runPrivileged(plan.steps, ui, {
     ...(deps.privileged === undefined ? {} : { deps: deps.privileged }),
-    title: "Install and start the supervised daemon?",
+    title: "Install and start the supervised session?",
     preamble: [
       `Installs ${plan.service.path} as ${installed}, then enables and restarts it.`,
+      ...(plan.herdrUnit === undefined
+        ? []
+        : [
+            `Also installs the fleet session unit as ${installedHerdr} with the account's login shell so panes`,
+            "run it instead of dash, then enables and restarts it.",
+            ...(plan.herdrConfigTarget === undefined
+              ? []
+              : [
+                  `And merges [terminal] default_shell into ${plan.herdrConfigTarget} as you, before that restart,`,
+                  "so the session server reads the pinned shell the moment it comes up.",
+                ]),
+          ]),
       "The unit runs as the account that staged it; nothing here changes that.",
+      "",
+      liveWorkers === 0
+        ? "No workers are live — nothing to drain before the restart."
+        : `Pauses new claims and drains ${liveWorkers} live worker(s) to idle first — this can take minutes per worker — then restarts the units and resumes dispatch.`,
     ],
+    beforeRun,
   });
   if (outcome.kind === "declined") return { kind: "declined", wrote };
   if (outcome.kind === "failed") {
+    // Restore the preceding dispatch state: a restart that failed is still a
+    // finished operator action whose units may or may not be up, so the fleet
+    // must not sit paused for it. A throw from the drain itself (a timeout or
+    // a superseded fence) never reaches here — it propagates and leaves
+    // dispatch paused, fail-closed, exactly as upgrade does.
+    if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
     return { kind: "failed", reason: `${outcome.step.title} exited ${outcome.exitCode}` };
   }
+  if (fence !== undefined && !fence.initialPaused) drainDeps.setPaused(false, fence.scope.pauseKey);
   ui.notify(`Installed and started ${STAGED_SERVICE_NAME}.`, "info");
   return { kind: "installed", wrote };
 }
 
+/**
+ * The pause/drain fence's default accessors, overridable per test through
+ * {@link InstallDeps.drain.deps}. Mirrors `upgrade.ts`'s {@link DrainDeps}
+ * wiring, except the pause names `setup host` as the actor and the drain
+ * reports progress through the same UI surface.
+ */
+function hostInstallDrainDeps(ui: WizardUi, deps: InstallDeps): DrainDeps {
+  if (deps.drain?.deps !== undefined) return deps.drain.deps;
+  return {
+    snapshot: statusSnapshot,
+    layers: fleetLayers,
+    projectNames: () => loadConfig().projects.map((p) => p.name),
+    daemonIdentity: () => {
+      const daemon = livingDaemon();
+      return daemon === undefined
+        ? { running: false }
+        : { running: true, project: daemon.project, generation: `${daemon.pid}@${daemon.startedAt}` };
+    },
+    pauseState: (project) => pauseInstance(project),
+    setPaused: (v, project) => setPaused(v, { source: "setup host", reason: "setup host, draining" }, project),
+    sleep: Bun.sleep,
+    log: (message) => ui.notify(message, "info"),
+  };
+}
+
+/**
+ * Attribute the runs the restart is about to kill: any run still live when the
+ * session host restarts underneath it is that restart's casualty, not a failed
+ * attempt, so its row records the actor. The classifier reads this as an
+ * administrative kill that costs no budget and requeues (#512). The fence
+ * drains to idle first, so in the healthy path this records nothing — it is
+ * the safety net for a claim that raced the drain or a worker the drain could
+ * not bring to idle.
+ */
+function attributeRestartKills(
+  scope: UpgradeScope,
+  project: string,
+  store: Pick<Store, "liveRuns" | "updateRun"> | undefined,
+): void {
+  const attribute = (owner: Pick<Store, "liveRuns" | "updateRun">): void => {
+    for (const selector of scope.selectors) {
+      for (const run of owner.liveRuns(selector ?? project)) {
+        owner.updateRun(run.id, { lastError: "admin restart: setup host (draining)" });
+      }
+    }
+  };
+  if (store !== undefined) {
+    attribute(store);
+    return;
+  }
+  const path = dbPath();
+  // No store yet — a host that has never dispatched — means nothing to attribute
+  // and nothing a fresh open would do but create an empty database.
+  if (!existsSync(path)) return;
+  const owner = openStore(path);
+  try {
+    attribute(owner);
+  } finally {
+    owner.close();
+  }
+}
+
 /** Fleet-wide worker sum when config is loadable; otherwise this project's caps. */
 function hostInstallWorkers(caps: Caps): number {
   try {
@@ -138,6 +373,22 @@ function hostInstallWorkers(caps: Caps): number {
   }
 }
 
+/**
+ * Whether this host configures more than one project. The recovery unit is
+ * host-global and must not encode one project's name when several are
+ * configured, so the plan scopes RECOVER_PROJECT by this (#510/#530). A
+ * loadable config is the truth; when it cannot be read (a not-yet-configured
+ * box), there is no evidence of several projects and the named case stays
+ * unambiguous — the same fallback shape as {@link hostInstallWorkers}.
+ */
+function hostMultiProject(): boolean {
+  try {
+    return loadConfig().projects.length > 1;
+  } catch {
+    return false;
+  }
+}
+
 export interface GraphInstallOptions extends InstallDeps {
   /**
    * Skip the seeding step. The timer is still installed and enabled, and the
@@ -219,6 +470,15 @@ export async function runGraphInstall(
   const blocked = linuxOnly(options);
   const { service, timer } = unitPaths(options.unitDir ?? SYSTEMD_UNIT_DIR);
   const from = unitPaths(stateDir());
+  // The units systemd already reads, compared to what staging just wrote — the
+  // same distinction setup-host.ts draws between `installedAction` and
+  // `service.action` (read the installed file, compare to the fresh content,
+  // "keep" vs "update"). "Installed" is not enough: a green timer serving a
+  // stale unit is the older bug, so when they differ the install runs. Only
+  // when both units already match does the plan omit them — then just the
+  // outstanding clone and seed remain.
+  const unitsCurrent =
+    blocked === undefined && sameFile(service, from.service) && sameFile(timer, from.timer);
   // 3. Install and enable, privileged. 4. Seed, in the SAME batch: the contract is
   //    one preview and one confirm, and a second confirm here also invented a
   //    third outcome — a declined seed — that neither the caller nor the
@@ -230,9 +490,13 @@ export async function runGraphInstall(
   const install: PrivilegedStep[] =
     blocked === undefined
       ? [
-          { title: "install the reindex unit and timer", argv: ["install", "-m", "0644", from.service, from.timer, join(options.unitDir ?? SYSTEMD_UNIT_DIR, "")] },
-          { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
-          { title: `enable ${REINDEX_UNIT}.timer`, argv: ["systemctl", "enable", "--now", `${REINDEX_UNIT}.timer`] },
+          ...(unitsCurrent
+            ? []
+            : [
+                { title: "install the reindex unit and timer", argv: ["install", "-m", "0644", from.service, from.timer, join(options.unitDir ?? SYSTEMD_UNIT_DIR, "")] },
+                { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
+                { title: `enable ${REINDEX_UNIT}.timer`, argv: ["systemctl", "enable", "--now", `${REINDEX_UNIT}.timer`] },
+              ]),
           ...seed,
         ]
       : [];
@@ -242,19 +506,19 @@ export async function runGraphInstall(
     return { kind: "staged", wrote: staged.written, reason: `${blocked} ${service}` };
   }
 
-  if (blocked !== undefined) {
-    ui.notify(`${blocked} ${service} and ${timer}`, "warning");
-    return { kind: "staged", wrote: staged.written, reason: `${blocked} ${service}` };
-  }
-
   const outcome = await runPrivileged([...clones, ...install], ui, {
     ...(options.privileged === undefined ? {} : { deps: options.privileged }),
-    title: "Clone, install and enable the code-graph timer?",
+    title: unitsCurrent ? "Clone the code-graph checkouts and seed them?" : "Clone, install and enable the code-graph timer?",
     preamble: [
       `Staged: ${staged.written.join(", ")}.`,
       ...(clones.length === 0
         ? ["Every indexed clone already exists."]
-        : [`${clones.length} clone(s) run as you; the unit install needs root.`]),
+        : [
+            `${clones.length} clone(s) run as you; ${
+              unitsCurrent ? "only the seed needs root — the reindex units already match." : "the unit install needs root."
+            }`,
+          ]),
+      ...(unitsCurrent ? ["The reindex units are already installed and current — nothing to install."] : []),
       `Indexer: ${prereqs.indexer ?? "on PATH"}.`,
     ],
   });