omp-conductor 0.15.10 → 0.15.12

package/src/setup-host.ts

@@ -1,10 +1,16 @@
-import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
+import { chmodSync, existsSync, lstatSync, mkdirSync, readFileSync, readlinkSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync, type Stats } from "node:fs";
 import { spawnSync } from "node:child_process";
 import { homedir, userInfo } from "node:os";
-import { dirname, join } from "node:path";
+import { dirname, join, relative } from "node:path";
 import { configPath, loadConfig, resolveCaps, stateDir } from "./config.ts";
 import { isPaused, runDaemon, statusSnapshot, type StatusSnapshot } from "./daemon.ts";
-import { DEFAULT_FLEET_AGENT_NAME, DEFAULT_HERDR_SESSION, resolveHerdrSessionWithBridge } from "./fleet.ts";
+import { ORCHESTRATOR_BRIEF_NAME } from "./brief-upgrade.ts";
+import {
+  DEFAULT_FLEET_AGENT_NAME,
+  DEFAULT_HERDR_SESSION,
+  DEFAULT_HERDR_UNIT,
+  resolveHerdrSessionWithBridge,
+} from "./fleet.ts";
 import {
   DEFAULT_PORT,
   healthCheck,
@@ -21,12 +27,41 @@ import {
   tickConfigMatchesProject,
   type TickConfig,
 } from "./orchestrator-tick.ts";
+import { formatStep, type PrivilegedStep } from "./privileged.ts";
 import type { Caps, ConductorConfig, ProjectConfig } from "./types.ts";
 
 export const DEFAULT_TICK_INTERVAL_SECONDS = 900;
 export const STAGED_SERVICE_NAME = "omp-conductor.service";
 export const SYSTEMD_UNIT_DIR = "/etc/systemd/system";
 
+/**
+ * The one recovery unit every fleet unit's `OnFailure=` points at (#485).
+ *
+ * A plain (non-template) unit on purpose: both fleet units name the same
+ * oneshot, and the playbook it runs determines which unit failed from live
+ * systemd state rather than trusting an instance name it cannot verify. The
+ * unit itself never carries `OnFailure=` — the one hard bound on
+ * recovery-triggering-recovery.
+ */
+export const RECOVER_SERVICE_NAME = "omp-conductor-recover.service";
+
+/** The playbook file shipped in the package's `systemd/` directory. */
+const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh";
+
+/**
+ * Where `setup host` installs the recovery playbook so the unit's ExecStart is
+ * a stable absolute path that survives package upgrades. /usr/local/sbin is
+ * root-executed operational tooling, on PATH for root on every target host.
+ */
+export const RECOVER_SCRIPT_INSTALL_PATH = "/usr/local/sbin/omp-conductor-recover";
+
+/**
+ * The symlink the session cwd loads as its brief. omp auto-loads `AGENTS.md`
+ * from the session cwd, so the fleet pane's cwd needs a link at this name
+ * resolving to the composed {@link ORCHESTRATOR_BRIEF_NAME}.
+ */
+export const AGENTS_BRIEF_NAME = "AGENTS.md";
+
 export type PlannedWrite<T> = {
   path: string;
   action: "create" | "update" | "keep";
@@ -34,9 +69,103 @@ export type PlannedWrite<T> = {
   value: T;
 };
 
+/**
+ * The `AGENTS.md` symlink setup places in a project's fleet cwd so the pane's
+ * session auto-loads the composed brief instead of staying brief-less.
+ *
+ * {@link action} is what the plan will *do*, compared against what sits at
+ * {@link path}:
+ *  - `create` — no entry yet, link it;
+ *  - `update` — a symlink occupies {@link path} but points elsewhere, replace it;
+ *  - `keep` — an already-correct symlink, leave untouched (idempotent re-run);
+ *  - `skip` — a regular file occupies {@link path}, never overwrite an
+ *    operator's file (see {@link skippedReason}).
+ */
+export interface BriefLinkPlan {
+  /** `AGENTS.md` inside the project's fleet cwd ({@link tickCwdForProject}). */
+  path: string;
+  /** Relative target inside the symlink — what the link resolves to. */
+  target: string;
+  action: "create" | "update" | "keep" | "skip";
+  /** Why {@link action} is `skip`; present only then. */
+  skippedReason?: string;
+}
+
+/**
+ * Why a host-global install (no `--project`) wrote no per-project tail, and
+ * what the operator does about it.
+ */
+export interface HostRuntimeNoProject {
+  /** The project-scoped files the host-global install deliberately left unwritten. */
+  skipped: readonly string[];
+  /** The command that writes them: `setup host` naming one project. */
+  how: string;
+}
+
 export interface HostRuntimePlan {
   service: PlannedWrite<string>;
+  /**
+   * The staged Herdr session-server unit. Absent when Herdr is not installed
+   * ({@link ServiceRuntime.herdr} is `undefined`) — a host driving Herdr some
+   * other way must stay supported. `upgrade` and `start` treat
+   * {@link DEFAULT_HERDR_UNIT} as a first-class dependency, and this is the
+   * rendering that provisions it (see {@link renderHerdrUnit}).
+   */
+  herdrUnit?: PlannedWrite<string>;
+  /**
+   * The fleet recovery oneshot (#485): staged always, because the daemon unit
+   * (which every fleet has) carries `OnFailure=` to it. Rendered by
+   * {@link renderRecoverUnit}.
+   */
+  recoverUnit: PlannedWrite<string>;
+  /**
+   * The recovery playbook body, staged from the shipped package file and
+   * installed to {@link RECOVER_SCRIPT_INSTALL_PATH} so the unit's ExecStart
+   * is a stable absolute path.
+   */
+  recoverScript: PlannedWrite<string>;
+  /**
+   * `[terminal] default_shell` staged into the session's herdr config, the
+   * belt-and-braces behind the unit's `SHELL=` (see {@link renderHerdrConfig}).
+   * Present exactly when {@link herdrUnit} is, unless the login shell is not a
+   * usable path ({@link usableHerdrShell}) or the rendered config would not
+   * parse — either one leaves {@link herdrConfigProblem} set and writes
+   * nothing.
+   *
+   * This is the **staged** copy, under the state directory like every other
+   * file the plan writes: herdr's live config is a file conductor does not own,
+   * so it is only replaced post-consent by the install step naming
+   * {@link herdrConfigTarget} (#513).
+   */
+  herdrConfig?: PlannedWrite<string>;
+  /** The live herdr config {@link herdrConfig} merges into, post-consent. */
+  herdrConfigTarget?: string;
+  /** Why no pane-shell key is planned (unusable login shell, …). */
+  herdrConfigProblem?: string;
   tick?: PlannedWrite<TickConfig>;
+  /** The `AGENTS.md` — composed-brief symlink — placed in the fleet cwd. */
+  briefLink?: BriefLinkPlan;
+  /**
+   * Set when an install ran with no project named (a host-global install):
+   * the per-project tail was deliberately not written. Names the files and
+   * how to write them, so the operator is told what is missing and how to get
+   * it rather than silently installing host-global units that leave a project
+   * brief-less and tick-less.
+   */
+  noProject?: HostRuntimeNoProject;
+  /**
+   * The exact privileged steps this plan will run, in order — recovery first,
+   * so the daemon unit that follows names a recovery unit systemd already has.
+   * This is the single list: {@link installCommands} and `runHostInstall` are
+   * both derived from it, so a step can never appear in the printed plan and
+   * be absent from the run (#509).
+   */
+  steps: readonly PrivilegedStep[];
+  /**
+   * The same steps as `sudo …` shell lines, for humans to read. A projection
+   * of {@link steps} rather than a parallel list, so nothing executes a step
+   * the printed plan does not show.
+   */
   installCommands: readonly string[];
   cliSource: "global" | "plugin";
   /** Absolute path of the unit systemd actually reads. */
@@ -54,6 +183,12 @@ export interface HostRuntimePlan {
    * the installed unit matched.
    */
   installedAction: PlannedWrite<string>["action"];
+  /**
+   * True when every file the privileged install steps would write is already
+   * at its destination with the current bytes. `runHostInstall` uses it to
+   * make a no-op re-run of `setup host` neither restage nor restart anything.
+   */
+  currentInstall: boolean;
 }
 
 export interface ServiceRuntime {
@@ -65,6 +200,21 @@ export interface ServiceRuntime {
   packageCli: string;
   conductorHome: string;
   telegramStateDir: string;
+  /**
+   * The account's login shell. Rendered into the Herdr session unit as
+   * `Environment=SHELL=` (and into the herdr config as `[terminal]
+   * default_shell`), so every pane in the fleet session runs this shell instead
+   * of the `/bin/sh` Herdr falls back to when `$SHELL` is unset (the fleet's
+   * dash-pane symptom, #456). A systemd service has no `$SHELL`, so this is
+   * load-bearing, not decoration.
+   */
+  shell: string;
+  /**
+   * Absolute path of the `herdr` binary on this host, or `undefined` when Herdr
+   * is not installed. When absent, setup stages no Herdr unit and no pane-shell
+   * config — a host driving Herdr some other way stays supported (#456).
+   */
+  herdr?: string;
   /**
    * Non-default Herdr session to pin into the unit (#394). Omitted when the
    * host uses {@link DEFAULT_HERDR_SESSION}: an always-present line would show
@@ -93,10 +243,6 @@ function systemdPath(value: string): string {
   return value.replaceAll("%", "%%");
 }
 
-function shellQuote(value: string): string {
-  return `'${value.replaceAll("'", "'\\''")}'`;
-}
-
 function actionFor(path: string, content: string): PlannedWrite<string>["action"] {
   if (!existsSync(path)) return "create";
   try {
@@ -252,6 +398,8 @@ function defaultServiceRuntime(
   const home = homedir();
   const bun = process.execPath;
   const globalCli = Bun.which("omp-conductor");
+  const herdr = Bun.which("herdr");
+  const loginShell = userInfo().shell;
   const pathParts = [dirname(bun), ...(process.env["PATH"] ?? "").split(":")].filter(
     (value, index, all) => value.length > 0 && all.indexOf(value) === index,
   );
@@ -264,6 +412,10 @@ function defaultServiceRuntime(
     packageCli: join(import.meta.dir, "cli.ts"),
     conductorHome: dirname(configPath()),
     telegramStateDir,
+    shell: loginShell ?? "/bin/sh",
+    // Only when Herdr is actually installed: a host running Herdr some other
+    // way (or none yet) gets no Herdr unit rendered, ever.
+    ...(herdr === null ? {} : { herdr }),
     // Only pin a non-default session. Default hosts keep a unit with no
     // HERDR_SESSION line so setup/upgrade do not invent drift (#394).
     ...(herdrSession !== DEFAULT_HERDR_SESSION ? { herdrSession } : {}),
@@ -289,6 +441,10 @@ export function renderDaemonService(runtime: ServiceRuntime, totalWorkers: numbe
     "Documentation=https://github.com/TerrifiedBug/conductor",
     "After=network-online.target",
     "Wants=network-online.target",
+    // #485: a crash-looped daemon used to sit in `failed` indefinitely, in
+    // silence — no unit was watching it. The recovery oneshot runs outside
+    // this unit's cgroup, collects evidence durably, and pages tier-2.
+    `OnFailure=${RECOVER_SERVICE_NAME}`,
     "",
     "[Service]",
     "Type=simple",
@@ -304,7 +460,12 @@ export function renderDaemonService(runtime: ServiceRuntime, totalWorkers: numbe
       : [`Environment=${systemdQuote(`HERDR_SESSION=${runtime.herdrSession}`)}`]),
     `WorkingDirectory=${systemdPath(stateDir())}`,
     `ExecStart=${command.map(systemdQuote).join(" ")}`,
-    "Restart=on-failure",
+    // #546: restart on any exit — clean, signalled or crashed — except an
+    // explicit `systemctl stop`, which systemd records as intentional and does
+    // not undo. A stray SIGTERM (a worker's `bun test`) then costs seconds of
+    // downtime instead of leaving the fleet down until a human. A crash loop
+    // still trips the start-limit burst and reaches `OnFailure=` above.
+    "Restart=always",
     "SuccessExitStatus=0 143",
     "MemoryAccounting=yes",
     `MemoryMax=${memoryMax}`,
@@ -315,6 +476,161 @@ export function renderDaemonService(runtime: ServiceRuntime, totalWorkers: numbe
   ].join("\n");
 }
 
+/**
+ * The Herdr session-server unit, in the same shape as the daemon unit.
+ *
+ * The fleet's 24/7 guarantee (`upgrade`'s restart + pane-recovery leg, `start`,
+ * the recovery plugin) treats {@link DEFAULT_HERDR_UNIT} as a first-class
+ * dependency, yet nothing rendered it — every operator hand-wrote it from a bare
+ * `ExecStart=` line. This provisions it, and it is where the dash-pane bug dies:
+ * a systemd service has no `$SHELL`, so without `Environment=SHELL=` Herdr
+ * resolved every pane as `/bin/sh`→dash with no readline and no `.bashrc`. The
+ * account's login shell is pinned here, and again in the herdr config
+ * ({@link renderHerdrConfig}) as belt and braces.
+ */
+export function renderHerdrUnit(runtime: ServiceRuntime): string {
+  if (runtime.herdr === undefined) throw new Error("cannot render a herdr unit without a herdr binary");
+  const session = runtime.herdrSession ?? DEFAULT_HERDR_SESSION;
+  const configPath = join(runtime.home, ".config", "herdr", "config.toml");
+  return [
+    "[Unit]",
+    "Description=herdr fleet session server",
+    "Documentation=https://github.com/TerrifiedBug/conductor",
+    "After=network-online.target",
+    "Wants=network-online.target",
+    // #485: the fleet's two units are each other's only plausible watcher and
+    // neither watched the other. The recovery oneshot is the watcher, and it
+    // must fire for the orchestrator host too — a dead herdr means no tick, so
+    // nobody else would ever report it.
+    `OnFailure=${RECOVER_SERVICE_NAME}`,
+    "",
+    "[Service]",
+    "Type=simple",
+    `User=${runtime.username}`,
+    `Environment=${systemdQuote(`HOME=${runtime.home}`)}`,
+    `Environment=${systemdQuote(`PATH=${runtime.path}`)}`,
+    `Environment=${systemdQuote(`HERDR_CONFIG_PATH=${configPath}`)}`,
+    `Environment=${systemdQuote(`SHELL=${runtime.shell}`)}`,
+    `WorkingDirectory=${systemdPath(runtime.conductorHome)}`,
+    `ExecStart=${systemdQuote(runtime.herdr)} --session ${systemdQuote(session)} server`,
+    // #546: same exposure as the daemon — restart on any exit except an
+    // explicit `systemctl stop`; a crash loop still trips the start limit.
+    "Restart=always",
+    "RestartSec=5",
+    "",
+    "[Install]",
+    "WantedBy=multi-user.target",
+    "",
+  ].join("\n");
+}
+
+/**
+ * The fleet recovery unit (#485): a oneshot every fleet unit's `OnFailure=`
+ * names.
+ *
+ * It runs as root, outside the fleet units' cgroups and process trees, so the
+ * restart it performs can never take it down, and it carries an explicit PATH —
+ * on a typical host the non-interactive ssh PATH lacks `omp`, `herdr` and
+ * `omp-conductor`, and the playbook needs all three.
+ *
+ * The bounds are part of the unit text so `doctor` drift sees them:
+ * `StartLimitIntervalSec`/`StartLimitBurst` cap failed recovery starts at the
+ * systemd layer (the playbook keeps its own durable consecutive-attempt
+ * counter, agreeing on the same window), and there is deliberately **no**
+ * `OnFailure=` of its own and no `[Install]` — nothing activates recovery
+ * except the two fleet units' failure, so recovery can never re-trigger
+ * recovery.
+ */
+export function renderRecoverUnit(runtime: ServiceRuntime, projectName: string | undefined): string {
+  // `RECOVER_PROJECT` is how the playbook addresses the tier-2 escalation it
+  // enqueues. It is a host-global unit installed once for the whole box, so a
+  // static per-project value is only unambiguous on a single-project host — a
+  // multi-project host (or a no-project, host-global install) leaves it unset
+  // and reports without a `--project` rather than guessing one (#510/#530).
+  return [
+    "[Unit]",
+    "Description=omp-conductor fleet recovery (OnFailure handler)",
+    "Documentation=https://github.com/TerrifiedBug/conductor",
+    // The oneshot bound: a recovery that fails twice in the window stops
+    // trying, and the playbook's durable counter uses the same window.
+    "StartLimitIntervalSec=1800",
+    "StartLimitBurst=2",
+    "",
+    "[Service]",
+    "Type=oneshot",
+    `Environment=${systemdQuote(`PATH=${runtime.path}`)}`,
+    `Environment=${systemdQuote(`OMP_CONDUCTOR_HOME=${runtime.conductorHome}`)}`,
+    `Environment=${systemdQuote(`RECOVER_STATE_DIR=${stateDir()}`)}`,
+    ...(projectName === undefined ? [] : [`Environment=${systemdQuote(`RECOVER_PROJECT=${projectName}`)}`]),
+    `ExecStart=${RECOVER_SCRIPT_INSTALL_PATH}`,
+    "",
+  ].join("\n");
+}
+
+/**
+ * Whether a login-shell value may be pinned into herdr's config at all.
+ *
+ * `userInfo().shell` degrades to the literal `unknown` where the passwd entry
+ * cannot be read (the same root as #511's drift check), and `unknown` is not a
+ * path — a pane told to exec it dies on start, which is a stopped fleet. Only
+ * an absolute path that exists on this host qualifies; anything else must mean
+ * the caller writes **no** key and lets herdr's own `$SHELL → /bin/sh →
+ * passwd` fallback apply.
+ */
+export function usableHerdrShell(shell: string): boolean {
+  if (!shell.startsWith("/")) return false;
+  try {
+    return existsSync(shell);
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Idempotently put `[terminal] default_shell` into a herdr config, preserving
+ * everything else verbatim.
+ *
+ * Belt-and-braces behind the unit's `Environment=SHELL=`: a pane comes up as the
+ * account's login shell even if an operator later edits the unit and drops the
+ * `SHELL` line. Herdr falls back `$SHELL → /bin/sh` and skips the passwd entry,
+ * so without this the only guarantee would be the unit's environment.
+ *
+ * A shell that is not a usable executable path ({@link usableHerdrShell})
+ * writes **no** key at all — the file passes through byte-identical, because
+ * "pin the shell" must never mean "write the word `unknown`".
+ */
+export function renderHerdrConfig(shell: string, existing?: string): string {
+  const text = existing ?? "";
+  if (!usableHerdrShell(shell)) return text;
+
+  const quoted = `"${shell.replaceAll("\\", "\\\\").replaceAll('"', '\\"')}"`;
+  const line = `default_shell = ${quoted}`;
+
+  // One anchored, comment-aware match, used for both the guard and the replace
+  // so they can never disagree: a *real* key starts its line (after whitespace),
+  // while `# default_shell = …` starts with a comment and is not a key. The
+  // old guard was unanchored, so it matched a commented line the anchored
+  // replace did not — a config whose only mention was a comment took the
+  // "replace" branch, replaced nothing, and silently wrote nothing at all.
+  // Any existing real key keeps its position and gets the new value in place —
+  // even when the value already matches, so a re-run stays idempotent instead
+  // of duplicating the key.
+  const keyLine = /^([ \t]*default_shell[ \t]*=\s*).*$/m;
+  if (keyLine.test(text)) {
+    return text.replace(keyLine, `$1${quoted}`);
+  }
+
+  const lines = text.split("\n");
+  const table = lines.findIndex((l) => /^\[terminal\]\s*$/.test(l));
+  if (table === -1) {
+    const body = text.replace(/\s+$/, "");
+    return `${body.length > 0 ? body + "\n" : ""}[terminal]\n${line}\n`;
+  }
+  // The table exists but has no default_shell; drop the key right under it.
+  lines.splice(table + 1, 0, line);
+  return lines.join("\n");
+}
+
 function tickSearchRoots(project: ProjectConfig): string[] {
   const roots = [stateDir(), dirname(project.workspaceRoot), project.workspaceRoot];
   return roots.filter((root, index) => roots.indexOf(root) === index);
@@ -373,6 +689,47 @@ export function tickCwdForProject(project: ProjectConfig): string {
   }
 }
 
+/**
+ * Where this project's `AGENTS.md` link should point and what setup will do
+ * about it. The link lives in the **fleet cwd** — the directory the pane runs
+ * in — pointing at the composed brief under `workspaceRoot`, by a relative
+ * target so the tree stays movable.
+ *
+ * A fleet whose tick config predates per-project roots keeps its cwd at the
+ * state dir, so the link is `<stateDir>/AGENTS.md -> worktrees/ORCHESTRATOR.md`;
+ * a brand-new project links `AGENTS.md -> ORCHESTRATOR.md` in its own cwd.
+ *
+ * Never a clobber: a symlink already at the brief is left alone, a stale
+ * symlink is replaceable, and a *regular file* is an operator's own brief —
+ * reported and left untouched.
+ */
+function planBriefLink(project: ProjectConfig): BriefLinkPlan {
+  const path = join(tickCwdForProject(project), AGENTS_BRIEF_NAME);
+  const target = relative(dirname(path), join(project.workspaceRoot, ORCHESTRATOR_BRIEF_NAME));
+  let st;
+  try {
+    st = lstatSync(path);
+  } catch {
+    // ENOENT (no link yet) and any unreadable path (no permission on a parent)
+    // both collapse to a best-effort create; a refused filesystem surfaces as
+    // a warning at write time.
+    return { path, target, action: "create" };
+  }
+  if (st.isSymbolicLink()) {
+    return {
+      path,
+      target,
+      action: readlinkSync(path, "utf8") === target ? "keep" : "update",
+    };
+  }
+  return {
+    path,
+    target,
+    action: "skip",
+    skippedReason: `an existing file at ${path} is not ours to overwrite; left untouched`,
+  };
+}
+
 function planTick(project: ProjectConfig, telegramStateDir: string): PlannedWrite<TickConfig> {
   const found = findProjectTick(project);
   const existing = found === undefined ? undefined : { path: found.path, config: found.config };
@@ -405,7 +762,14 @@ function planTick(project: ProjectConfig, telegramStateDir: string): PlannedWrit
 }
 
 export function planHostRuntime(
-  project: ProjectConfig,
+  /**
+   * The project the per-project tail (tick config, brief link, the recovery
+   * unit's RECOVER_PROJECT) is planned for. `undefined` plans the host-global
+   * units only — the same staged paths and systemd destinations, one shared
+   * daemon — and a `noProject` note naming the per-project files it left for
+   * a named run (#530).
+   */
+  project: ProjectConfig | undefined,
   _caps: Caps,
   telegramStateDir: string,
   runtime: ServiceRuntime = defaultServiceRuntime(telegramStateDir),
@@ -413,6 +777,17 @@ export function planHostRuntime(
   // Defaulting through loadConfig() would throw in install tests that stage
   // files before a config exists, and would hide a missing total at the call site.
   totalWorkers: number = 1,
+  // The directory the privileged steps install units into. `runHostInstall`
+  // injects a test-hermetic directory so the executed argv is assertable; the
+  // real install always resolves to `/etc/systemd/system`.
+  unitDir: string = SYSTEMD_UNIT_DIR,
+  // Where the recovery playbook installs; injectable like {@link unitDir} so
+  // the idempotency gate is testable without touching the real `/usr/local/sbin`.
+  recoverScriptInstallPath: string = RECOVER_SCRIPT_INSTALL_PATH,
+  // True when the host configures more than one project. The recovery unit is
+  // host-global, so on a multi-project host it must not encode one project's
+  // name — and a no-project install never does (#510/#530).
+  multiProject: boolean = false,
 ): HostRuntimePlan {
   const servicePath = join(stateDir(), STAGED_SERVICE_NAME);
   const serviceContent = renderDaemonService(runtime, totalWorkers);
@@ -422,21 +797,201 @@ export function planHostRuntime(
     content: serviceContent,
     value: serviceContent,
   };
-  const installedPath = join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME);
+  const herdrUnitPath = join(stateDir(), DEFAULT_HERDR_UNIT);
+  const herdrConfigPath = join(runtime.home, ".config", "herdr", "config.toml");
+  // The pane-shell config is staged like every other file; the live herdr
+  // config — a file conductor does not own — is only written by the install
+  // step, after consent (#513). A declined `setup host` must leave herdr's
+  // config byte-for-byte untouched, and the merge must land before the unit
+  // restart that accompanies it.
+  const herdrConfigStagedPath = join(stateDir(), "herdr-config.toml");
+  let herdrUnit: PlannedWrite<string> | undefined;
+  let herdrConfig: PlannedWrite<string> | undefined;
+  let herdrConfigTarget: string | undefined;
+  let herdrConfigProblem: string | undefined;
+  // The recovery playbook ships in this package's systemd/ dir; the same
+  // bytes go into the staged copy, so version control is the single source.
+  const recoverScriptPath = join(stateDir(), RECOVER_SCRIPT_FILE);
+  let recoverScriptContent: string;
+  try {
+    recoverScriptContent = readFileSync(join(import.meta.dir, "..", "systemd", RECOVER_SCRIPT_FILE), "utf8");
+  } catch (err) {
+    throw new Error(
+      `cannot read the recovery playbook at ${join(import.meta.dir, "..", "systemd", RECOVER_SCRIPT_FILE)} — it ships with this package, and setup cannot stage it without it: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  if (runtime.herdr !== undefined) {
+    const unitContent = renderHerdrUnit(runtime);
+    herdrUnit = {
+      path: herdrUnitPath,
+      action: actionFor(herdrUnitPath, unitContent),
+      content: unitContent,
+      value: unitContent,
+    };
+    // Belt and braces: read the session's herdr config so `[terminal]
+    // default_shell` merges into whatever the operator already has, preserving
+    // it verbatim. A missing or unreadable config starts from scratch.
+    let existing: string | undefined;
+    try {
+      existing = readFileSync(herdrConfigPath, "utf8");
+    } catch {
+      existing = undefined;
+    }
+    // An unresolvable login shell (userInfo() renders the literal "unknown")
+    // plans nothing and says why: never a placeholder, and never a value that
+    // is not an executable path — herdr's own `$SHELL → /bin/sh → passwd`
+    // fallback applies instead.
+    if (!usableHerdrShell(runtime.shell)) {
+      herdrConfigProblem =
+        `login shell ${JSON.stringify(runtime.shell)} is not an absolute path that exists on this host — ` +
+        "no [terminal] default_shell is written, so herdr's own $SHELL → /bin/sh → passwd fallback applies";
+    } else {
+      const configContent = renderHerdrConfig(runtime.shell, existing);
+      // The rendered file is parse-checked before anything may replace herdr's
+      // live config. An input that is already broken (a duplicated key, say)
+      // renders broken — the replace touches every real key, so two keys stay
+      // two keys — and writing that file is exactly the stopped-fleet outcome
+      // this must prevent. A parse failure aborts the whole plan here, before
+      // a single file has been staged.
+      if (actionFor(herdrConfigPath, configContent) !== "keep") {
+        try {
+          Bun.TOML.parse(configContent);
+        } catch (err) {
+          throw new Error(
+            `refusing to write herdr's config at ${herdrConfigPath}: it would not parse ` +
+              `(${err instanceof Error ? err.message : String(err)}). Nothing has been staged — ` +
+              "fix the file by hand (herdr's `config check` names the line), then re-run setup host.",
+          );
+        }
+      }
+      herdrConfig = {
+        path: herdrConfigStagedPath,
+        action: actionFor(herdrConfigStagedPath, configContent),
+        content: configContent,
+        value: configContent,
+      };
+      herdrConfigTarget = herdrConfigPath;
+    }
+  }
+  const installedPath = join(unitDir, STAGED_SERVICE_NAME);
+  const installedHerdr = join(unitDir, DEFAULT_HERDR_UNIT);
+  const recoverUnitPath = join(stateDir(), RECOVER_SERVICE_NAME);
+  // The recovery unit is host-global: one shared unit, installed once. A
+  // static RECOVER_PROJECT is only legitimate when there is exactly one
+  // project to be unambiguous about — a multi-project host (or a no-project
+  // install) leaves it unset so the escalation reports without attributing a
+  // sibling's crash to one project (#510/#530).
+  const recoverProject = project === undefined || multiProject ? undefined : project.name;
+  const recoverUnitContent = renderRecoverUnit(runtime, recoverProject);
+  const recoverUnit: PlannedWrite<string> = {
+    path: recoverUnitPath,
+    action: actionFor(recoverUnitPath, recoverUnitContent),
+    content: recoverUnitContent,
+    value: recoverUnitContent,
+  };
+  const recoverScript: PlannedWrite<string> = {
+    path: recoverScriptPath,
+    action: actionFor(recoverScriptPath, recoverScriptContent),
+    content: recoverScriptContent,
+    value: recoverScriptContent,
+  };
+  // Recovery first: the daemon unit that follows names it in OnFailure=, so
+  // the restart below must never point at a unit systemd cannot load. This is
+  // the single list `runHostInstall` executes and {@link installCommands}
+  // renders from, so no step can be in one and missing from the other (#509).
+  const installSteps: PrivilegedStep[] = [
+    {
+      title: "install the recovery playbook",
+      argv: ["install", "-m", "0755", recoverScriptPath, recoverScriptInstallPath],
+    },
+    {
+      title: `install ${RECOVER_SERVICE_NAME}`,
+      argv: ["install", "-m", "0644", recoverUnitPath, join(unitDir, RECOVER_SERVICE_NAME)],
+    },
+    { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
+    {
+      title: `install ${STAGED_SERVICE_NAME}`,
+      argv: ["install", "-m", "0644", servicePath, installedPath],
+    },
+    { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
+    { title: `enable ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "enable", STAGED_SERVICE_NAME] },
+    { title: `restart ${STAGED_SERVICE_NAME}`, argv: ["systemctl", "restart", STAGED_SERVICE_NAME] },
+    // The herdr session server, provisioned alongside the daemon (#456). Only
+    // when herdr is installed and the plan therefore staged a unit.
+    ...(herdrUnit === undefined
+      ? []
+      : [
+          {
+            title: `install ${DEFAULT_HERDR_UNIT}`,
+            argv: ["install", "-m", "0644", herdrUnitPath, installedHerdr],
+          },
+          { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
+          { title: `enable ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "enable", DEFAULT_HERDR_UNIT] },
+          // The pane-shell merge lands *before* the restart it accompanies, so
+          // the session server comes up with the pinned shell, not the stale
+          // config. It runs without sudo on purpose: the file is the fleet
+          // account's own (the escalation guard guarantees setup runs as that
+          // account), and a root-owned copy would break herdr's next rewrite.
+          // Nothing writes herdr's live config until this consent-gated step.
+          ...(herdrConfig === undefined || actionFor(herdrConfigPath, herdrConfig.content) === "keep"
+            ? []
+            : [
+                {
+                  title: `write the pane shell into ${herdrConfigPath}`,
+                  argv: ["install", "-m", "0644", herdrConfigStagedPath, herdrConfigPath],
+                  unprivileged: true,
+                },
+              ]),
+          { title: `restart ${DEFAULT_HERDR_UNIT}`, argv: ["systemctl", "restart", DEFAULT_HERDR_UNIT] },
+        ]),
+  ];
+  // Everything the privileged steps install is already at its destination with
+  // the current bytes, so a re-run of `setup host` has nothing to install and
+  // nothing to restart. The herdr unit is absent on a host without herdr, and
+  // an absent unit that would not be provisioned is nothing to do.
+  const installedAction = actionFor(installedPath, serviceContent);
+  const currentInstall =
+    installedAction === "keep" &&
+    actionFor(join(unitDir, RECOVER_SERVICE_NAME), recoverUnitContent) === "keep" &&
+    actionFor(recoverScriptInstallPath, recoverScriptContent) === "keep" &&
+    (herdrUnit === undefined || actionFor(installedHerdr, herdrUnit.content) === "keep") &&
+    // The pane-shell file is a destination like the units: a re-run with all
+    // units current but the config merge still pending must not report
+    // "nothing to install" and skip the very write the plan exists to make.
+    (herdrConfig === undefined || actionFor(herdrConfigPath, herdrConfig.content) === "keep");
   return {
     service,
-    ...(project.escalation.orchestrator === "external"
-      ? { tick: planTick(project, telegramStateDir) }
-      : {}),
-    installCommands: [
-      `sudo install -m 0644 ${shellQuote(servicePath)} ${shellQuote(installedPath)}`,
-      "sudo systemctl daemon-reload",
-      `sudo systemctl enable ${STAGED_SERVICE_NAME}`,
-      `sudo systemctl restart ${STAGED_SERVICE_NAME}`,
-    ],
+    // The herdr unit and pane-shell config stand and fall together: no herdr, no
+    // session to supervise, nothing for a pane shell to belong to. (The pane
+    // shell is omitted too when the login shell is unusable or the rendered
+    // config would not parse — those plans carry {@link herdrConfigProblem}.)
+    ...(herdrUnit === undefined ? {} : { herdrUnit }),
+    ...(herdrConfig === undefined ? {} : { herdrConfig, herdrConfigTarget }),
+    ...(herdrConfigProblem === undefined ? {} : { herdrConfigProblem }),
+    recoverUnit,
+    recoverScript,
+    ...(project === undefined
+      ? {
+          // Host-global install: the per-project tail entities are not written,
+          // and the plan says exactly which files and how to write them.
+          noProject: {
+            skipped: [TICK_CONFIG_FILE, AGENTS_BRIEF_NAME],
+            how: "re-run `omp-conductor setup host <NAME>` (or --project NAME) to write them for one project",
+          },
+        }
+      : {
+          briefLink: planBriefLink(project),
+          ...(project.escalation.orchestrator === "external"
+            ? { tick: planTick(project, telegramStateDir) }
+            : {}),
+        }),
+    steps: installSteps,
+    // For humans to read; the executed form is {@link steps}, in argv.
+    installCommands: installSteps.map((s) => formatStep(s)),
     cliSource: runtime.cli === undefined ? "plugin" : "global",
     installedPath,
-    installedAction: actionFor(installedPath, serviceContent),
+    installedAction,
+    currentInstall,
   };
 }
 
@@ -445,6 +1000,16 @@ export function formatHostRuntimePlan(plan: HostRuntimePlan): string {
     "host runtime",
     `  service        ${plan.service.action} ${plan.service.path}`,
     `  daemon entry   ${plan.cliSource === "global" ? "installed omp-conductor CLI" : "current installed plugin"}`,
+    `  recovery       ${plan.recoverUnit.action} ${plan.recoverUnit.path}`,
+    `  recovery exec  ${plan.recoverScript.action} ${plan.recoverScript.path} -> ${RECOVER_SCRIPT_INSTALL_PATH}`,
+    ...(plan.herdrUnit === undefined
+      ? ["  herdr session  skipped — herdr not installed on this host"]
+      : [
+          `  herdr session  ${plan.herdrUnit.action} ${plan.herdrUnit.path}`,
+          ...(plan.herdrConfig === undefined
+            ? [`  pane shell     skipped — ${plan.herdrConfigProblem ?? "login shell unusable"}`]
+            : [`  pane shell     ${plan.herdrConfig.action} ${plan.herdrConfig.path} ([terminal] default_shell)`]),
+        ]),
   ];
   if (plan.tick !== undefined) {
     lines.push(
@@ -456,6 +1021,19 @@ export function formatHostRuntimePlan(plan: HostRuntimePlan): string {
   } else {
     lines.push("  heartbeat      embedded orchestrator — no external tick config");
   }
+  if (plan.noProject !== undefined) {
+    lines.push(
+      `  per-project    skipped (no project named): ${plan.noProject.skipped.join(", ")}`,
+      `                 ${plan.noProject.how}`,
+    );
+  }
+  if (plan.briefLink !== undefined) {
+    lines.push(
+      plan.briefLink.action === "skip"
+        ? `  brief link     ${plan.briefLink.path} — ${plan.briefLink.skippedReason}`
+        : `  brief link     ${plan.briefLink.action} ${plan.briefLink.path} -> ${plan.briefLink.target}`,
+    );
+  }
   lines.push("  install        staged only; the final result prints the systemd install commands");
   return lines.join("\n");
 }
@@ -474,17 +1052,77 @@ function atomicWrite(path: string, content: string, mode: number): void {
   }
 }
 
-export function writeHostRuntime(plan: HostRuntimePlan): string[] {
-  const written: string[] = [];
+/** What {@link writeHostRuntime} wrote, plus anything it could not. */
+export interface HostRuntimeWrite {
+  /** Paths that were (re)written. */
+  wrote: string[];
+  /** Best-effort failures the caller should surface; setup continues regardless. */
+  warnings: string[];
+}
+
+export function writeHostRuntime(plan: HostRuntimePlan): HostRuntimeWrite {
+  const wrote: string[] = [];
+  const warnings: string[] = [];
   if (plan.service.action !== "keep") {
     atomicWrite(plan.service.path, plan.service.content, 0o644);
-    written.push(plan.service.path);
+    wrote.push(plan.service.path);
+  }
+  if (plan.herdrUnit !== undefined && plan.herdrUnit.action !== "keep") {
+    atomicWrite(plan.herdrUnit.path, plan.herdrUnit.content, 0o644);
+    wrote.push(plan.herdrUnit.path);
+  }
+  if (plan.herdrConfig !== undefined && plan.herdrConfig.action !== "keep") {
+    atomicWrite(plan.herdrConfig.path, plan.herdrConfig.content, 0o644);
+    wrote.push(plan.herdrConfig.path);
+  }
+  if (plan.recoverUnit.action !== "keep") {
+    atomicWrite(plan.recoverUnit.path, plan.recoverUnit.content, 0o644);
+    wrote.push(plan.recoverUnit.path);
+  }
+  if (plan.recoverScript.action !== "keep") {
+    atomicWrite(plan.recoverScript.path, plan.recoverScript.content, 0o755);
+    wrote.push(plan.recoverScript.path);
   }
   if (plan.tick !== undefined && plan.tick.action !== "keep") {
     atomicWrite(plan.tick.path, plan.tick.content, 0o600);
-    written.push(plan.tick.path);
+    wrote.push(plan.tick.path);
+  }
+  // `skip` is a regular file the plan already told the operator would be left
+  // alone; `keep` is an already-correct link. Only create/update touch disk —
+  // and never on the plan's word: the entry can change across the consent gap
+  // (an operator may drop a regular file where a stale symlink was planned for
+  // replacement). Re-lstat at write time and act on *current* state, so a file
+  // that appears mid-flight is preserved, never unlinked by a stale plan.
+  // A host-global plan (no project) has no briefLink to write at all.
+  if (plan.briefLink !== undefined && (plan.briefLink.action === "create" || plan.briefLink.action === "update")) {
+    const { path, target } = plan.briefLink;
+    try {
+      mkdirSync(dirname(path), { recursive: true });
+      let st: Stats | undefined;
+      try {
+        st = lstatSync(path);
+      } catch {
+        st = undefined; // ENOENT — nothing there yet; also best-effort if a parent refuses.
+      }
+      if (st !== undefined && !st.isSymbolicLink()) {
+        // A regular file occupies the path (present at plan time, appeared, or
+        // swapped in since). Never clobber an operator's file: warn and carry on.
+        warnings.push(`${path} is a regular file, not ours to overwrite; AGENTS.md brief link skipped`);
+      } else if (st !== undefined && readlinkSync(path, "utf8") === target) {
+        // Already the correct link — leave it untouched, however the plan read it.
+      } else {
+        // Missing, or a stale symlink: (re)link it. Only a *current* symlink is
+        // removed; an operator's file that appears mid-flight survives the write.
+        if (st !== undefined) rmSync(path, { force: true });
+        symlinkSync(target, path);
+        wrote.push(path);
+      }
+    } catch (err) {
+      // Best effort: a filesystem that refuses symlinks must not fail setup.
+      warnings.push(`could not link ${path} -> ${target}: ${err instanceof Error ? err.message : String(err)}`);
+    }
   }
-  return written;
+  return { wrote, warnings };
 }
 
 export interface SetupSmokeResult {