omp-conductor 0.16.1 → 0.16.2

package/src/setup-host.ts

@@ -211,6 +211,14 @@ export interface HostRuntimePlan {
    * the installed unit matched.
    */
   installedAction: PlannedWrite<string>["action"];
+  /**
+   * The installed destinations whose live bytes differ from this version's
+   * render — exactly the files the privileged steps would rewrite. `upgrade`
+   * names them after a release that re-rendered a template, and
+   * {@link currentInstall} is `true` iff this list is empty, so the two can
+   * never disagree (#598).
+   */
+  drift: readonly string[];
   /**
    * True when every file the privileged install steps would write is already
    * at its destination with the current bytes. `runHostInstall` uses it to
@@ -1285,18 +1293,27 @@ export function planHostRuntime(
   // nothing to restart. The herdr unit is absent on a host without herdr, and
   // an absent unit that would not be provisioned is nothing to do.
   const installedAction = actionFor(installedPath, serviceContent);
+  // The same comparison as a destination list, so `upgrade` can name the
+  // drifted files and `currentInstall` can never disagree with that list: a
+  // plan is current exactly when no installed destination differs from the
+  // render.
+  const drift: string[] = [];
+  const noteDrift = (path: string, content: string | undefined): void => {
+    if (content !== undefined && actionFor(path, content) !== "keep") drift.push(path);
+  };
+  noteDrift(installedPath, serviceContent);
+  noteDrift(join(unitDir, RECOVER_SERVICE_NAME), recoverUnitContent);
+  noteDrift(recoverScriptInstallPath, recoverScriptContent);
+  if (herdrUnit !== undefined) noteDrift(installedHerdr, herdrUnit.content);
+  // The pane-shell file is a destination like the units: a plan with all
+  // units current but the config merge still pending must not report
+  // "nothing to install" and skip the very write it exists to make.
+  if (herdrConfig !== undefined) noteDrift(herdrConfigPath, herdrConfig.content);
+  // Same for the herdr-conductor config.env: a pending env merge is pending
+  // work, not an already-current install.
+  if (herdrEnv !== undefined) noteDrift(herdrEnvTarget, herdrEnv.content);
   const currentInstall =
-    installedAction === "keep" &&
-    actionFor(join(unitDir, RECOVER_SERVICE_NAME), recoverUnitContent) === "keep" &&
-    actionFor(recoverScriptInstallPath, recoverScriptContent) === "keep" &&
-    (herdrUnit === undefined || actionFor(installedHerdr, herdrUnit.content) === "keep") &&
-    // The pane-shell file is a destination like the units: a re-run with all
-    // units current but the config merge still pending must not report
-    // "nothing to install" and skip the very write the plan exists to make.
-    (herdrConfig === undefined || actionFor(herdrConfigPath, herdrConfig.content) === "keep") &&
-    // Same for the herdr-conductor config.env: pending env merge is pending
-    // work, not an already-current install.
-    (herdrEnv === undefined || actionFor(herdrEnvTarget, herdrEnv.content) === "keep");
+    drift.length === 0;
   return {
     service,
     // The herdr unit and pane-shell config stand and fall together: no herdr, no
@@ -1349,6 +1366,7 @@ export function planHostRuntime(
     cliSource: runtime.cli === undefined ? "plugin" : "global",
     installedPath,
     installedAction,
+    drift,
     currentInstall,
   };
 }

package/src/setup-wizard.ts

@@ -2005,7 +2005,7 @@ export async function setup(
   areaArg?: AmendAreaId,
   probes: SetupProbes = DEFAULT_PROBES,
   apply: SetupApplyDeps = DEFAULT_APPLY,
-): Promise<void> {
+): Promise<boolean> {
   const path = configPath();
   // A config that exists but does not parse is a fault to report, never
   // something to quietly replace: overwriting it would delete every project it
@@ -2021,7 +2021,7 @@ export async function setup(
   } catch (err) {
     if (!(err instanceof Cancelled)) throw err;
     ui.notify("Setup cancelled — nothing was changed.", "info");
-    return;
+    return false;
   }
   // `answers` is mutable below: the review loop re-asks one area at a time and
   // replaces only the fields that area owns, so everything else carries across.
@@ -2035,7 +2035,7 @@ export async function setup(
         `Run \`gh auth refresh -s repo,project\`, then run setup again.`,
       "error",
     );
-    return;
+    return false;
   }
 
   // Drafted here — after the interview, before the plan — because the plan's single
@@ -2183,7 +2183,7 @@ export async function setup(
       .join(" ");
 
   let plan = await derive();
-  if (plan === undefined) return;
+  if (plan === undefined) return false;
   ui.notify(planBlock(plan), "info");
   // -------- the consent gate (#417): a review loop, not a yes/no --------
   //
@@ -2206,12 +2206,12 @@ export async function setup(
     );
     if (choice === undefined) {
       ui.notify("Setup cancelled — nothing was changed, and the answers were discarded.", "info");
-      return;
+      return false;
     }
     if (choice === REVIEW_APPLY) break;
     if (choice === REVIEW_EXIT) {
       ui.notify("Setup exited without applying — the answers were discarded and nothing was changed.", "info");
-      return;
+      return false;
     }
     if (choice === REVIEW_REVIEW) {
       ui.notify(planBlock(plan), "info");
@@ -2259,7 +2259,7 @@ export async function setup(
       proseDrafted = true;
     }
     const redrafted = await derive();
-    if (redrafted === undefined) return;
+    if (redrafted === undefined) return false;
     plan = redrafted;
     ui.notify(`Edited "${area.label}" — the plan below is recomputed from the new answers.`, "info");
     ui.notify(planBlock(plan), "info");
@@ -2302,7 +2302,7 @@ export async function setup(
       // enables the timer, which does not touch the running daemon.
       await offerCodeGraph(plan.project, ui, apply.graphInstall);
       ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
-      return;
+      return false;
     }
     // No live workers: still prefer an explicit reload when this run *added* a
     // neighbour — auto-restart would bounce every other project's heartbeat for
@@ -2342,7 +2342,7 @@ export async function setup(
         "warning",
       );
       ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
-      return;
+      return false;
     }
   }
   apply.resume(plan.project.name);
@@ -2409,4 +2409,5 @@ export async function setup(
   // to create/verify the herdr pane. Added projects especially — the CLI wrote
   // tick + config but cannot start a herdr agent (#319).
   ui.notify(formatHerdrHandoff(plan.project, plan.nextConfig), "info");
+  return true;
 }

package/src/upgrade-verify.ts

@@ -382,6 +382,19 @@ function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequ
   }
 }
 
+/**
+ * Doctor findings that only a privileged `setup host` can clear — the systemd
+ * units (and the recovery pair they name), and the herdr pane-shell key. A
+ * release that re-renders any of those templates fails them on every host
+ * whose `setup host` has not been re-run, so they are not rollback-worthy
+ * regression evidence: the upgrade is deliberately unable to perform the
+ * privileged install, and rolling back over drift it cannot clear would
+ * strand the fleet on the previous release with the same outstanding `setup
+ * host` (#598). The verifier escalates them instead — the doctor check's
+ * detail and the verify report name the drift and its fix.
+ */
+const PRIVILEGED_INSTALL_DRIFT = new Set(["systemd-unit", "systemd-recovery", "herdr-resume"]);
+
 /**
  * The independent checks the returning process runs. Each compares a live
  * fact with the journal's record of the world the install began in:
@@ -392,7 +405,9 @@ function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequ
  *    same predicate the in-process upgrade used, so a fleet that was armed
  *    stays armed and a live pane stays live;
  *  - `health` — the daemon that came back answers `/healthz`;
- *  - `doctor` — no failing finding.
+ *  - `doctor` — no failing finding beyond the privileged-install drift
+ *    ({@link PRIVILEGED_INSTALL_DRIFT}) only `setup host` can clear, which is
+ *    escalated rather than rolled back.
  */
 export async function runUpgradeChecks(
   deps: UpgradeVerifyDeps,
@@ -423,10 +438,17 @@ export async function runUpgradeChecks(
 
   const doctor = await deps.doctor(deps.projectName);
   const failing = doctor.findings.filter((finding) => finding.status === "fail");
+  const drift = failing.filter((finding) => PRIVILEGED_INSTALL_DRIFT.has(finding.id));
+  const regression = failing.filter((finding) => !PRIVILEGED_INSTALL_DRIFT.has(finding.id));
   checks.push({
     name: "doctor",
-    ok: failing.length === 0,
-    detail: failing.length === 0 ? `status ${doctor.status}` : `${failing[0]!.id}: ${failing[0]!.summary}`,
+    ok: regression.length === 0,
+    detail:
+      regression.length > 0
+        ? `${regression[0]!.id}: ${regression[0]!.summary}`
+        : drift.length > 0
+          ? `status ${doctor.status}; ${drift.map((f) => f.id).join(", ")} — privileged-install drift, run \`omp-conductor setup host\` from the fleet account`
+          : `status ${doctor.status}`,
   });
 }
 

package/src/upgrade.ts

@@ -14,9 +14,9 @@ import { livingDaemon, restartDaemon } from "./lifecycle.ts";
 import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";
 import {
-  STAGED_SERVICE_NAME,
   planHostRuntime,
-  writeHostRuntime,
+  totalConfiguredWorkers,
+  type HostRuntimePlan,
 } from "./setup-host.ts";
 import {
   appendJournal,
@@ -95,6 +95,17 @@ export interface UpgradeDeps {
   sleep(ms: number): Promise<void>;
   env: NodeJS.ProcessEnv;
   log(message: string): void;
+  /**
+   * The host runtime this version's package renders, compared against what is
+   * actually installed: the daemon and herdr units, the recovery unit and its
+   * playbook at {@link RECOVER_SCRIPT_INSTALL_PATH}, the herdr pane-shell
+   * config and the herdr-conductor `config.env`. `upgrade` prints
+   * {@link HostRuntimePlan.drift} after a release that re-rendered those
+   * files, so the operator is told that `setup host` is owed before a later
+   * `doctor` has to discover it. Read-only: the plan renders and compares,
+   * it never writes a host file.
+   */
+  hostRuntime(): HostRuntimePlan;
   /**
    * The durable journal sink for the detached fleet installer (#486). Wired
    * to the state-dir journal by `upgrade-install`/`upgrade-rollback`; absent
@@ -135,6 +146,13 @@ export const DEFAULT_DEPS: UpgradeDeps = {
   restartDaemon: async () => {
     await restartDaemon({});
   },
+  // The bare host-global plan — the same render the advisory's `setup host`
+  // command would install: no per-project tail, the recovery unit encoding no
+  // one project's name, and FLEET_CWDS derived from every configured project.
+  hostRuntime: () => {
+    const cfg = loadConfig();
+    return planHostRuntime(undefined, cfg.defaults, telegramStateDir(), undefined, totalConfiguredWorkers(cfg));
+  },
   sleep: Bun.sleep,
   env: process.env,
   log: (message) => process.stdout.write(`${message}\n`),
@@ -821,6 +839,37 @@ export async function rollbackUpgrade(
   if (failures.length > 0) throw new Error(failures.join("; "));
 }
 
+/**
+ * Name the host-runtime destinations this version's render no longer matches,
+ * after a successful upgrade — the installed units, the recovery playbook at
+ * {@link RECOVER_SCRIPT_INSTALL_PATH}, the herdr pane-shell config and the
+ * herdr-conductor `config.env`. A release that re-renders any of those files
+ * silently invalidates the installed copies; telling the operator here — with
+ * the exact command — is what stops the drift from surviving until a later
+ * `doctor` finds it (#598).
+ *
+ * The plan *is* the comparison: {@link UpgradeDeps.hostRuntime} renders every
+ * destination against disk, so a current host prints nothing and a drifted one
+ * names exactly the files only a privileged `setup host` can refresh. This
+ * never mutates a host file and never fails the upgrade — an unreadable host
+ * state is a warning line, not a reason to roll a successful install back.
+ */
+function logHostRuntimeDrift(deps: UpgradeDeps): void {
+  let plan: HostRuntimePlan;
+  try {
+    plan = deps.hostRuntime();
+  } catch (err) {
+    deps.log(
+      `host runtime: could not compare against this version's render — ${err instanceof Error ? err.message : String(err)}`,
+    );
+    return;
+  }
+  if (plan.drift.length === 0) return;
+  deps.log("host runtime:");
+  for (const path of plan.drift) deps.log(`  ${path} differs from this version's render`);
+  deps.log("fix: run `omp-conductor setup host` from the fleet account to re-install the host units");
+}
+
 export async function upgradeConductor(
   options: UpgradeOptions = {},
   overrides: Partial<UpgradeDeps> = {},
@@ -871,6 +920,11 @@ export async function upgradeConductor(
         detail: "all three surfaces and every brief were already pinned to the target release",
       });
     }
+    // Packages are current, but the host units they render may be behind: the
+    // same drift this advisory names after an install applies when a release
+    // re-rendered a template since the operator's last `setup host`. Silent on
+    // a host whose runtime matches (#598).
+    logHostRuntimeDrift(deps);
     return {
       previousVersion: surfaces.cliVersion,
       ...release,
@@ -1118,6 +1172,11 @@ export async function upgradeConductor(
     throw new Error(`upgrade failed: ${failure}; previous installation restored; dispatch remains paused`);
   }
 
+  // The install landed; name any host-runtime destination this version no
+  // longer renders identically, so the operator knows `setup host` is owed
+  // before a later `doctor` reports it as drift (#598).
+  logHostRuntimeDrift(deps);
+
   return {
     previousVersion: surfaces.cliVersion,
     ...release,

package/systemd/omp-conductor-recover.sh

@@ -273,7 +273,7 @@ fresh_rollback_snapshot() {
   newest=$(ls -t "$BACKUP_ROOT"/config.json.pre-upgrade-* 2>/dev/null | head -n 1 || true)
   [[ -n $newest ]] || return 1
   now=$(epoch_now)
-  ts=$(stat -c %Y "$newest" 2>/dev/null || true)
+  ts=$(stat -c %Y "$newest" 2>/dev/null || stat -f %m "$newest" 2>/dev/null || true)
   [[ -n $ts && $((now - ts)) -le $ROLLBACK_AGE_S ]] || return 1
   printf '%s\n' "$newest"
 }

package/systemd/recover-unit-test.sh

@@ -126,14 +126,14 @@ unit_state() { # <case-dir> <unit> <state|absent>
 }
 
 # A pre-upgrade snapshot `upgrade` durably keeps (configBackupDir()):
-# config.json.pre-upgrade-<ts>. The test pass the age in seconds.
+# config.json.pre-upgrade-<ts>. The test passes the age in seconds.
 write_snapshot() { # <case-dir> <age-seconds>
   local d="$1" age="${2:-0}" now
   now=$(date +%s)
   local snap="$d/state/backups/config/config.json.pre-upgrade-$now"
   printf '%s\n' '{"preUpgrade":true}' >"$snap"
   if (( age > 0 )); then
-    touch -d "@$(( now - age ))" "$snap"
+    perl -e 'utime $ARGV[1], $ARGV[1], $ARGV[0] or die "utime: $!"' "$snap" "$(( now - age ))"
   fi
   printf '%s\n' "$snap"
 }