omp-conductor 0.15.10 → 0.15.12

package/src/upgrade.ts

@@ -12,13 +12,28 @@ import {
   type FleetLayers,
 } from "./fleet.ts";
 import { livingDaemon, restartDaemon } from "./lifecycle.ts";
-import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, writeConfigRaw } from "./config.ts";
+import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";
 import {
   STAGED_SERVICE_NAME,
   planHostRuntime,
   writeHostRuntime,
 } from "./setup-host.ts";
+import {
+  appendJournal,
+  readUpgradeJournal,
+  upgradeJournalPath,
+  type UpgradeJournalEntry,
+} from "./upgrade-journal.ts";
+import {
+  enqueueUpgradeReport,
+  pendingUpgradeRequest,
+  reportProject,
+  requestProgression,
+  runCommand,
+  type PendingUpgradeRequest,
+  type UpgradeCommandResult,
+} from "./upgrade-verify.ts";
 
 const PACKAGE = "omp-conductor";
 const HERDR_PLUGIN = "herdr-conductor";
@@ -28,15 +43,17 @@ const DRAIN_POLL_MS = 5_000;
 const RECOVERY_POLL_MS = 2_000;
 const RECOVERY_ATTEMPTS = 30;
 
-export interface UpgradeCommandResult {
-  code: number;
-  stdout: string;
-  stderr: string;
-}
-
 export interface UpgradeOptions {
   version?: string;
   project?: string;
+  /**
+   * Run as the detached fleet installer (#486): journal every surface the
+   * transaction touches, skip the in-process verification and leave dispatch
+   * paused — the first tick of the post-restart daemon verifies version,
+   * /healthz, ticks, pane and doctor, and closes or rolls back on that
+   * evidence. Only the transient unit's `upgrade-install` entry sets this.
+   */
+  detached?: boolean;
 }
 
 export interface UpgradeResult {
@@ -79,25 +96,18 @@ export interface UpgradeDeps {
   sleep(ms: number): Promise<void>;
   env: NodeJS.ProcessEnv;
   log(message: string): void;
+  /**
+   * The durable journal sink for the detached fleet installer (#486). Wired
+   * to the state-dir journal by `upgrade-install`/`upgrade-rollback`; absent
+   * on the human `upgrade` path, whose own process is its witness. The engine
+   * journals each surface boundary so the post-restart tick can verify against
+   * a record it did not write.
+   */
+  journal?(entry: UpgradeJournalEntry): void;
 }
 
-async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
-  try {
-    const child = Bun.spawn([command, ...args], {
-      stdin: "ignore",
-      stdout: "pipe",
-      stderr: "pipe",
-      env: process.env,
-    });
-    const stdout = new Response(child.stdout).text();
-    const stderr = new Response(child.stderr).text();
-    const code = await child.exited;
-    return { code, stdout: await stdout, stderr: await stderr };
-  } catch (err) {
-    return { code: 127, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
-  }
-}
-
+/** One command run to completion with captured output. Exported so the
+ *  post-restart verifier and the rollback unit run processes the same way. */
 export const DEFAULT_DEPS: UpgradeDeps = {
   run: runCommand,
   snapshot: statusSnapshot,
@@ -301,13 +311,39 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
  * exactly this transaction's pause while leaving any per-project operator hold
  * standing.
  */
-interface UpgradeScope {
+export interface UpgradeScope {
   /** Project selectors to address the project-aware deps with, never empty. */
   selectors: readonly (string | undefined)[];
   /** The sentinel this transaction pauses and resumes. */
   pauseKey: string | undefined;
 }
 
+/**
+ * The subset of {@link UpgradeDeps} the pause/drain fence needs. Factored out
+ * so a caller that only wants the fence — `setup host` — can provide just
+ * these members instead of a full install-and-verify deps object. Every
+ * {@link UpgradeDeps} is a `DrainDeps`.
+ */
+export interface DrainDeps {
+  /** Live worker count per project selector; `undefined` = the one configured project. */
+  snapshot(project?: string): { liveWorkers: number };
+  /** The fleet's current dispatch/daemon layers for one project selector. */
+  layers(project?: string): FleetLayers;
+  /** Every configured project name, in config order (#389). */
+  projectNames(): readonly string[];
+  /**
+   * The daemon the transaction targets: running or not, its project when it
+   * recorded one, and a `generation` identifying the exact instance (#377).
+   */
+  daemonIdentity(): { running: boolean; project?: string; generation?: string };
+  /** The active pause sentinel as an instance, when one is readable (#377). */
+  pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
+  /** Write or clear the durable pause sentinel for the scope's project. */
+  setPaused(value: boolean, project?: string): void;
+  sleep(ms: number): Promise<void>;
+  log(message: string): void;
+}
+
 /**
  * Decides which projects a transaction covers, and refuses the requests that
  * cannot be honoured truthfully.
@@ -326,7 +362,11 @@ interface UpgradeScope {
  * A single-configured-project host keeps the historical bare selector, so its
  * behaviour is unchanged whether or not the daemon recorded its name.
  */
-function resolveScope(deps: UpgradeDeps, verb: "upgrade" | "restart", project?: string): UpgradeScope {
+export function resolveScope(
+  deps: DrainDeps,
+  verb: "upgrade" | "restart" | "setup host",
+  project?: string,
+): UpgradeScope {
   let configured: readonly string[] = [];
   try {
     configured = deps.projectNames();
@@ -362,8 +402,8 @@ function resolveScope(deps: UpgradeDeps, verb: "upgrade" | "restart", project?:
   return { selectors: [undefined], pauseKey: undefined };
 }
 
-async function waitForDrain(
-  deps: UpgradeDeps,
+export async function waitForDrain(
+  deps: DrainDeps,
   scope: UpgradeScope,
   deadlineAt?: number,
   stale?: () => string | undefined,
@@ -417,7 +457,7 @@ interface RestartBegun {
  * stopped by it.
  */
 function restartFenceProblem(
-  deps: UpgradeDeps,
+  deps: DrainDeps,
   scope: UpgradeScope,
   begun: RestartBegun,
 ): string | undefined {
@@ -440,6 +480,59 @@ function restartFenceProblem(
   return undefined;
 }
 
+/**
+ * Pause claims and drain live workers to idle — the destructive half of the
+ * trusted restart transaction, minus the restart. Exported so `setup host`
+ * runs the same fence `upgrade` and `restart` do, then performs its own unit
+ * installs instead of {@link drainAndRestart}'s daemon restart.
+ *
+ * Generation-scoped (#377) exactly as the full transaction is: the caller
+ * captures the restart-owned pause and the daemon generation, and the drain
+ * aborts if a `resume` lifts the pause, replaces the sentinel, or a newer
+ * daemon generation appears. On ANY throw after the pause the fleet stays
+ * paused (no resume in a catch), so a drain that outlives `timeoutMs` cannot
+ * silently resume dispatch over a wedged daemon. `timeoutMs` absent waits
+ * indefinitely, which is the `upgrade` posture.
+ */
+export async function pauseAndDrain(
+  deps: DrainDeps,
+  verb: "upgrade" | "restart" | "setup host",
+  o: { project?: string; timeoutMs?: number },
+): Promise<{ scope: UpgradeScope; initialPaused: boolean }> {
+  // Host-wide by default (#389): the daemon this restarts serves every
+  // configured project, so the drain counts every project's workers and the
+  // pause covers all of them.
+  const scope = resolveScope(deps, verb, o.project);
+  const initial = deps.layers(scope.pauseKey);
+  if (!initial.paused) deps.setPaused(true, scope.pauseKey);
+  // The lock must be provable NOW. A pause that cannot be read as an instance
+  // — sentinel malformed or unreadable — would make the whole fence fail open
+  // (nothing to compare against), so the transaction refuses before waiting
+  // on anything it cannot act upon either way.
+  const pauseToken = deps.pauseState(scope.pauseKey);
+  if (pauseToken === undefined) {
+    throw new Error(
+      `${verb} cancelled: cannot prove the ${verb}-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted`,
+    );
+  }
+  const begun: RestartBegun = {
+    pauseToken,
+    daemon: deps.daemonIdentity(),
+  };
+  const stale = () => restartFenceProblem(deps, scope, begun);
+  await waitForDrain(
+    deps,
+    scope,
+    o.timeoutMs === undefined ? undefined : Date.now() + o.timeoutMs,
+    stale,
+  );
+  // The drain completed; the world may have moved on while it did. Re-prove
+  // the pause and the generation an instant before the destructive call.
+  const cancelled = stale();
+  if (cancelled !== undefined) throw new Error(cancelled);
+  return { scope, initialPaused: initial.paused };
+}
+
 /**
  * Pause, drain, restart, restore — the trusted restart transaction, minus the
  * install/verify steps of {@link upgradeConductor}.
@@ -467,34 +560,9 @@ export async function drainAndRestart(
   deps: UpgradeDeps,
   o: { project?: string; timeoutMs: number },
 ): Promise<void> {
-  // Host-wide by default (#389): the daemon this restarts serves every
-  // configured project, so the drain counts every project's workers and the
-  // pause covers all of them.
-  const scope = resolveScope(deps, "restart", o.project);
-  const initial = deps.layers(scope.pauseKey);
-  if (!initial.paused) deps.setPaused(true, scope.pauseKey);
-  // The lock must be provable NOW. A pause that cannot be read as an instance
-  // — sentinel malformed or unreadable — would make the whole fence fail open
-  // (nothing to compare against), so the transaction refuses before waiting
-  // on anything it cannot act upon either way.
-  const pauseToken = deps.pauseState(scope.pauseKey);
-  if (pauseToken === undefined) {
-    throw new Error(
-      "restart cancelled: cannot prove the restart-owned pause — the active pause sentinel is unreadable or malformed; nothing was restarted",
-    );
-  }
-  const begun: RestartBegun = {
-    pauseToken,
-    daemon: deps.daemonIdentity(),
-  };
-  const stale = () => restartFenceProblem(deps, scope, begun);
-  await waitForDrain(deps, scope, Date.now() + o.timeoutMs, stale);
-  // The drain completed; the world may have moved on while it did. Re-prove
-  // the pause and the generation an instant before the destructive call.
-  const cancelled = stale();
-  if (cancelled !== undefined) throw new Error(cancelled);
+  const { scope, initialPaused } = await pauseAndDrain(deps, "restart", o);
   await deps.restartDaemon();
-  if (!initial.paused) deps.setPaused(false, scope.pauseKey);
+  if (!initialPaused) deps.setPaused(false, scope.pauseKey);
 }
 
 function recoveryProblem(
@@ -576,7 +644,10 @@ async function upgradeBriefs(deps: UpgradeDeps, briefs: readonly ScopedBrief[]):
   }
 }
 
-async function rollbackUpgrade(
+/** Restore every surface the upgrade touched, in reverse, from the identities
+ *  captured before the first install. Exported so the detached rollback unit
+ *  can restore what the detached installer recorded (#486). */
+export async function rollbackUpgrade(
   deps: UpgradeDeps,
   previous: InstalledSurfaces,
   scope: UpgradeScope,
@@ -736,6 +807,10 @@ export async function upgradeConductor(
   overrides: Partial<UpgradeDeps> = {},
 ): Promise<UpgradeResult> {
   const deps: UpgradeDeps = { ...DEFAULT_DEPS, ...overrides };
+  const detached = options.detached === true;
+  const journal = (entry: Omit<UpgradeJournalEntry, "at">, at = new Date()): void => {
+    deps.journal?.({ ...entry, at: at.toISOString() });
+  };
   if (deps.env["HERDR_ENV"] !== undefined) {
     throw new Error("run omp-conductor upgrade from a shell outside the target Herdr session");
   }
@@ -748,6 +823,18 @@ export async function upgradeConductor(
   const release = parseRegistry(
     (await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
   );
+  if (detached) {
+    // The durable request: written before anything can pause, so a unit that
+    // dies after this line leaves a journal the returning process can act on
+    // (roll back, resume, or report) instead of a paused fleet with no record.
+    journal({
+      kind: "request",
+      ok: true,
+      version: release.version,
+      gitHead: release.gitHead,
+      unit: deps.env["OMP_CONDUCTOR_UNIT"],
+    });
+  }
   const initial = deps.layers(scope.pauseKey);
   const surfaces = await inspectSurfaces(deps);
   const briefs: ScopedBrief[] = scope.selectors.map((selector) => ({
@@ -756,6 +843,15 @@ export async function upgradeConductor(
   }));
   const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
   if (!installNeeded && briefs.every((b) => b.current)) {
+    if (detached) {
+      journal({
+        kind: "outcome",
+        phase: "already-current",
+        ok: true,
+        version: release.version,
+        detail: "all three surfaces and every brief were already pinned to the target release",
+      });
+    }
     return {
       previousVersion: surfaces.cliVersion,
       ...release,
@@ -796,20 +892,51 @@ export async function upgradeConductor(
   }
   if (initial.herdr === "unknown") throw new Error("cannot determine whether herdr-fleet.service is active");
 
+  if (detached) {
+    // The durable record the returning process rolls back or verifies against:
+    // what was installed, what dispatch state the fleet began in, which pause
+    // sentinel this transaction owns, and the config snapshot for the restore.
+    journal({
+      kind: "snapshot",
+      ok: true,
+      version: release.version,
+      gitHead: release.gitHead,
+      unit: deps.env["OMP_CONDUCTOR_UNIT"],
+      previous: {
+        cliVersion: surfaces.cliVersion,
+        ompVersion: surfaces.ompVersion,
+        herdrSource: surfaces.herdrSource,
+      },
+      ...(preUpgradeBackup === undefined ? {} : { configBackup: preUpgradeBackup }),
+      pauseKey: scope.pauseKey,
+      initialPaused: initial.paused,
+      initial: {
+        dispatch: initial.dispatch,
+        ticks: initial.ticks,
+        pane: initial.pane,
+        herdr: initial.herdr,
+      },
+      selectors: scope.selectors.map((s) => s ?? null),
+    });
+  }
+
   deps.log(`target release: omp-conductor@${release.version} (${release.gitHead})`);
   if (!initial.paused) {
     deps.log("safety: pausing new issue claims");
     deps.setPaused(true, scope.pauseKey);
   }
+  journal({ kind: "phase", phase: "paused", surface: "dispatch", ok: true, version: release.version });
   deps.log("drain: waiting for live omp worker sessions");
   try {
     await waitForDrain(deps, scope);
   } catch (err) {
+    journal({ kind: "phase", phase: "drain", ok: false, version: release.version, detail: err instanceof Error ? err.message : String(err) });
     const failure = err instanceof Error ? err.message : String(err);
     throw new Error(
       `upgrade failed while draining workers: ${failure}; no installation started; dispatch remains paused`,
     );
   }
+  journal({ kind: "phase", phase: "drain", ok: true, version: release.version });
   deps.log("drain: complete");
 
   let installTouched = false;
@@ -821,8 +948,10 @@ export async function upgradeConductor(
       installTouched = true;
       deps.log(`install 1/3: Bun-global omp-conductor CLI → ${release.version}`);
       await mustRun(deps, "bun", ["add", "-g", `${PACKAGE}@${release.version}`]);
+      journal({ kind: "phase", phase: "install", surface: "cli", ok: true, version: release.version, gitHead: release.gitHead });
       deps.log(`install 2/3: omp plugin omp-conductor → ${release.version}`);
       await mustRun(deps, "omp", ["plugin", "install", `${PACKAGE}@${release.version}`]);
+      journal({ kind: "phase", phase: "install", surface: "omp", ok: true, version: release.version, gitHead: release.gitHead });
       if (surfaces.herdrSource?.startsWith("local:")) {
         await mustRun(deps, "herdr", ["plugin", "unlink", HERDR_PLUGIN]);
       }
@@ -835,6 +964,7 @@ export async function upgradeConductor(
         release.gitHead,
         "--yes",
       ]);
+      journal({ kind: "phase", phase: "install", surface: "herdr", ok: true, version: release.version, gitHead: release.gitHead });
     } else {
       deps.log("install: CLI, omp plugin, and Herdr plugin already pinned to the target release");
     }
@@ -842,16 +972,40 @@ export async function upgradeConductor(
     deps.log("brief: refreshing managed ORCHESTRATOR.md floor while preserving POLICY.md");
     briefChanged = true;
     await upgradeBriefs(deps, briefs);
+    journal({ kind: "phase", phase: "brief", surface: "brief", ok: true, version: release.version });
 
     if (initial.herdr === "active") {
       herdrReloadStarted = true;
       deps.log("reload: restarting herdr-fleet.service and recovering the orchestrator pane");
       await mustRun(deps, "systemctl", ["restart", HERDR_UNIT]);
+      journal({ kind: "phase", phase: "restart", surface: "herdr", ok: true, version: release.version });
     }
     if (initial.dispatch !== "stopped") {
       daemonReloadStarted = true;
       deps.log("reload: restarting the omp-conductor dispatch daemon");
       await deps.restartDaemon();
+      journal({ kind: "phase", phase: "restart", surface: "daemon", ok: true, version: release.version });
+    }
+
+    if (detached) {
+      // The installing process does not declare its own success: it journals
+      // that the transaction reached the reloads and stops, leaving dispatch
+      // paused exactly where it was. The first tick of the post-restart daemon
+      // verifies against this journal and is the only witness that closes the
+      // transaction (#486).
+      journal({
+        kind: "phase",
+        phase: "awaiting-verification",
+        ok: true,
+        version: release.version,
+        detail: "installed; the first tick after the restart verifies and restores dispatch",
+      });
+      return {
+        previousVersion: surfaces.cliVersion,
+        ...release,
+        alreadyCurrent: false,
+        dispatch: initial.dispatch,
+      };
     }
 
     deps.log("verify 1/2: package identities, services, pane, ticks, and paused dispatch");
@@ -885,10 +1039,21 @@ export async function upgradeConductor(
       deps.setPaused(true, scope.pauseKey);
     }
     const failure = err instanceof Error ? err.message : String(err);
+    journal({
+      kind: "phase",
+      phase: "failed",
+      ok: false,
+      version: release.version,
+      detail: failure,
+    });
     deps.log(`upgrade failed: ${failure}`);
+    const journalLog = (message: string): void => {
+      journal({ kind: "phase", phase: "rollback", ok: true, version: release.version, detail: message });
+      deps.log(message);
+    };
     try {
       await rollbackUpgrade(
-        deps,
+        { ...deps, log: journalLog },
         surfaces,
         scope,
         installTouched,
@@ -898,8 +1063,35 @@ export async function upgradeConductor(
         configBefore,
         preUpgradeBackup,
       );
+      journal({ kind: "outcome", phase: "rolled-back", ok: false, version: release.version, detail: failure });
+      enqueueUpgradeReport(
+        "tier2",
+        reportProject(scope.selectors),
+        `fleet upgrade to omp-conductor@${release.version} failed and was rolled back`,
+        [
+          `Reason: ${failure}`,
+          `Restored: cli=${surfaces.cliVersion}, omp=${surfaces.ompVersion ?? "missing"}, ` +
+            `herdr=${surfaces.herdrSource ?? "missing"}`,
+          ...(preUpgradeBackup === undefined ? [] : [`Pre-upgrade config snapshot: ${preUpgradeBackup}`]),
+          "Dispatch stays paused until the first tick after the restart records the rollback verified.",
+        ],
+        `upgrade:${release.version}:rolled-back`,
+      );
     } catch (rollbackErr) {
       const rollback = rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr);
+      journal({ kind: "outcome", phase: "rollback-failed", ok: false, version: release.version, detail: rollback });
+      enqueueUpgradeReport(
+        "tier2",
+        reportProject(scope.selectors),
+        `upgrade to omp-conductor@${release.version} failed and ITS ROLLBACK ALSO FAILED`,
+        [
+          `Failure: ${failure}`,
+          `Rollback failure: ${rollback}`,
+          "The fleet may be at mixed versions; do not resume dispatch.",
+          "Restore manually from the journal at " + upgradeJournalPath(stateDir()) + ".",
+        ],
+        `upgrade:${release.version}:rollback-failed`,
+      );
       throw new Error(
         `upgrade failed: ${failure}; rollback incomplete: ${rollback}; dispatch remains paused`,
       );
@@ -914,3 +1106,138 @@ export async function upgradeConductor(
     dispatch: initial.dispatch,
   };
 }
+
+// ---------------------------------------------------------------------------
+// The detached fleet installer (#486)
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the upgrade transaction as the detached fleet installer: `upgrade`'s
+ * engine with a journal wired to the state-dir journal and the in-process
+ * verification handed to the first tick after the restart. This is the
+ * `upgrade-install` command the transient unit runs — the one process a
+ * `systemctl restart herdr-fleet.service` and a daemon restart both pass by.
+ */
+export async function runDetachedUpgrade(options: UpgradeOptions = {}): Promise<UpgradeResult> {
+  return upgradeConductor(
+    { ...options, detached: true },
+    {
+      journal: (entry) => appendJournal(stateDir(), entry),
+    },
+  );
+}
+
+// ---------------------------------------------------------------------------
+// The detached rollback unit (#486)
+// ---------------------------------------------------------------------------
+
+export interface RollbackFromJournalResult {
+  /** The version the rollback restored the fleet to. */
+  restoredVersion: string;
+  /** The version the failed upgrade had targeted. */
+  failedVersion: string;
+}
+
+/**
+ * Restore every surface the failed upgrade touched, from the journal the
+ * detached installer wrote. This is the `upgrade-rollback` command the
+ * verifier's transient unit runs — the same trust boundary as the install:
+ * the process that rolls back is not the process that failed, and it acts
+ * only on identities the journal recorded before the first install touched
+ * anything.
+ *
+ * Scoped to the newest request's progression: an older, already-closed
+ * upgrade in the same append-only journal neither supplies the snapshot to
+ * restore nor the phases that say which surfaces were touched, and its
+ * terminal outcome cannot refuse this rollback.
+ */
+export async function rollbackFromJournal(
+  overrides: Partial<UpgradeDeps> = {},
+  root = stateDir(),
+): Promise<RollbackFromJournalResult> {
+  const deps: UpgradeDeps = { ...DEFAULT_DEPS, ...overrides };
+  const entries = readUpgradeJournal(root);
+  const progression = requestProgression(entries);
+  const request = pendingUpgradeRequest(entries);
+  if (progression === undefined || request === undefined) {
+    throw new Error("no upgrade request is journaled; nothing to roll back");
+  }
+  const snapshot = progression.find((entry) => entry.kind === "snapshot");
+  if (snapshot === undefined || snapshot.previous === undefined) {
+    throw new Error(
+      `upgrade ${request.version} has no durable pre-install snapshot; refusing to guess what to restore`,
+    );
+  }
+  const newestOutcome = [...progression].reverse().find((entry) => entry.kind === "outcome");
+  if (newestOutcome !== undefined && newestOutcome.phase !== "rollback-requested") {
+    // Already terminal (verified, rolled back in-process, aborted). A second
+    // rollback would fight the newer outcome.
+    throw new Error(
+      `upgrade ${request.version} already closed as "${newestOutcome.phase}"; nothing to roll back`,
+    );
+  }
+
+  const phases = progression.filter((entry) => entry.kind === "phase");
+  const scope: UpgradeScope = {
+    selectors: request.selectors,
+    pauseKey: request.pauseKey,
+  };
+  const previous: InstalledSurfaces = {
+    cliVersion: snapshot.previous.cliVersion,
+    ompVersion: snapshot.previous.ompVersion,
+    herdrSource: snapshot.previous.herdrSource,
+  };
+  let configBefore: string | undefined;
+  if (request.configBackup !== undefined && existsSync(request.configBackup)) {
+    try {
+      configBefore = readFileSync(request.configBackup, "utf8");
+    } catch {
+      configBefore = undefined;
+    }
+  }
+  const journalLog = (message: string): void => {
+    appendJournal(root, {
+      at: new Date().toISOString(),
+      kind: "phase",
+      phase: "rollback",
+      ok: true,
+      version: request.version,
+      detail: message,
+    });
+    process.stdout.write(`${message}\n`);
+  };
+  await rollbackUpgrade(
+    { ...deps, log: journalLog },
+    previous,
+    scope,
+    phases.some((entry) => entry.phase === "install"),
+    phases.some((entry) => entry.phase === "brief"),
+    phases.some((entry) => entry.phase === "restart" && entry.surface === "herdr"),
+    phases.some((entry) => entry.phase === "restart" && entry.surface === "daemon"),
+    configBefore,
+    request.configBackup,
+  );
+
+  const restoredVersion = previous.cliVersion;
+  appendJournal(root, {
+    at: new Date().toISOString(),
+    kind: "outcome",
+    phase: "rolled-back",
+    ok: false,
+    version: request.version,
+    detail: `restored ${previous.cliVersion} from the pre-install snapshot`,
+  });
+  enqueueUpgradeReport(
+    "tier2",
+    reportProject(request.selectors),
+    `fleet restored to omp-conductor@${restoredVersion} after the failed upgrade to ${request.version}`,
+    [
+      "The transient rollback unit restored every surface from the durable snapshot;",
+      "dispatch stays paused until the first tick after these restarts verifies the old version.",
+      `Pre-upgrade config snapshot: ${request.configBackup ?? "none recorded"}`,
+    ],
+    `upgrade:${request.version}:rolled-back`,
+  );
+  return { restoredVersion, failedVersion: request.version };
+}
+