omp-conductor 0.19.4 → 0.19.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.19.4",
+  "version": "0.19.5",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/src/commands/resume.ts

@@ -13,10 +13,15 @@
 
 import type { CommandContext } from "./context.ts";
 import { armState, clearPaneHaltIfResolvable, releaseHold } from "../fleet.ts";
+import { acknowledgeUpgradeRecovery } from "../upgrade-verify.ts";
 import { wakeDispatch } from "../wake.ts";
 
 export async function resumeCommand(ctx: CommandContext): Promise<void> {
 for (const project of ctx.targetProjects()) {
+  // `releaseHold(project)` also removes the host-global sentinel. Record an
+  // unresolved recovery acknowledgement first even when a project-local pause
+  // shadows that global fence; the helper is a no-op when nothing is owed.
+  acknowledgeUpgradeRecovery();
   releaseHold(project.name);
   const pin = clearPaneHaltIfResolvable(project.name);
   const arm = armState(project.name);

package/src/daemon.ts

@@ -58,7 +58,7 @@ import {
   setPaused,
 } from "./pause.ts";
 import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
-import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
+import { runCommand, upgradeRecoveryStatus, verifyPendingUpgrade, type UpgradeRecoveryStatus, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
 import { inspectSurfaces, type InstalledSurfaces } from "./upgrade.ts";
 import { checkTelegramFreshness, type TelegramFreshness } from "./telegram-freshness.ts";
 import { processStartTimeMs } from "./upgrade-verify.ts";
@@ -5295,6 +5295,7 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
     layers: (project) => fleetLayers(project),
     health: healthCheck,
     doctor: (name) => runDoctor(name),
+    surfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
     enqueue: (draft) => {
       try {
         d.store.enqueueReport(draft);
@@ -5304,6 +5305,21 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
     },
     pause: pauseInstance,
     resume: (project) => setPaused(false, undefined, project),
+    holdGlobal: () => {
+      if (pauseInstance() === undefined) {
+        setPaused(true, { source: "upgrade-recovery", reason: "upgrade recovery verification required" });
+      }
+    },
+    releaseGlobalRecovery: (expectedSince) => {
+      const pause = pauseInstance();
+      if (
+        pause?.source === "upgrade-recovery" &&
+        expectedSince !== undefined &&
+        pause.since === expectedSince
+      ) {
+        setPaused(false);
+      }
+    },
     launchRollback: async (version) => {
       const launched = await launchTransientUnit(
         (argv) => runCommand(argv[0]!, argv.slice(1)),
@@ -5333,7 +5349,11 @@ async function verifyPendingUpgradeTick(
   const verifyDeps = upgradeVerifyDepsFor(d);
   try {
     const outcome = await run(verifyDeps);
-    if (outcome.handled !== "none" && outcome.handled !== "already-closed") {
+    if (
+      outcome.handled !== "none" &&
+      outcome.handled !== "already-closed" &&
+      outcome.handled !== "recovery-owed"
+    ) {
       log(
         `upgrade journal handled: ${outcome.handled}` +
           `${outcome.version === undefined ? "" : ` for omp-conductor@${outcome.version}`}` +
@@ -6480,6 +6500,8 @@ export interface StatusSnapshot {
    * reading like a mistake (#220).
    */
   pauseReason?: string;
+  /** Unresolved host-wide upgrade recovery, rendered identically for every project. */
+  upgradeRecovery?: UpgradeRecoveryStatus;
   /**
    * The project's active self-expiring drain (#484): present only while a
    * fresh, valid drain record exists. Paused and drained are deliberately two
@@ -6674,9 +6696,12 @@ export function statusSnapshotFromStore(
   const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
   // Read once: `status` renders the degrade row off this while it is down.
   const orchestratorDown = store.orchestratorIncident(p.name);
-  // Read once: the provenance read touches the filesystem, and the renderer
-  // should never pay for it twice per status.
-  const reason = pauseProvenance(p.name)?.reason;
+  // The journal is the authoritative recovery lifecycle. Render unresolved
+  // recovery even when an operator/project pause currently wins provenance,
+  // or before the verifier has re-created a missing global fence.
+  const pause = pauseProvenance(p.name);
+  const reason = pause?.reason;
+  const upgradeRecovery = upgradeRecoveryStatus(readUpgradeJournal(stateDir()), stateDir());
   // Same cost discipline as the pause read: the drain record is a file read,
   // and the active-drain view is only built when one is actually fresh. The
   // read is observational — it never mutates the record — so a status read
@@ -6741,6 +6766,7 @@ export function statusSnapshotFromStore(
     stateDir: stateDir(),
     paused: isPaused(p.name),
     ...(reason === undefined ? {} : { pauseReason: reason }),
+    ...(upgradeRecovery === undefined ? {} : { upgradeRecovery }),
     ...(drain.kind === "active"
       ? {
           drain: {

package/src/dashboard/controls.ts

@@ -46,6 +46,7 @@ import { unblockIssue } from "../unblock.ts";
 import { makeTracker } from "../tracker/github.ts";
 import { dbPath, openStore } from "../store.ts";
 import { loadConfig, resolveCaps } from "../config.ts";
+import { acknowledgeUpgradeRecovery } from "../upgrade-verify.ts";
 import type { ProjectConfig } from "../types.ts";
 
 /** The source string every dashboard mutation attributes itself with. */
@@ -70,6 +71,7 @@ export interface ControlDeps {
   project: ProjectConfig;
   hold?: typeof hold;
   releaseHold?: typeof releaseHold;
+  acknowledgeUpgradeRecovery?: typeof acknowledgeUpgradeRecovery;
   disarmTicks?: typeof disarmTicks;
   /**
    * Forwards one request to the owning daemon's own HTTP surface. Returns
@@ -119,6 +121,7 @@ export function dashboardPause(body: unknown, d: ControlDeps): ControlOutcome {
 
 /** Clear the pause. Idempotent, exactly as `resume` is on the CLI. */
 export function dashboardResume(d: ControlDeps): ControlOutcome {
+  (d.acknowledgeUpgradeRecovery ?? acknowledgeUpgradeRecovery)();
   (d.releaseHold ?? releaseHold)(d.project.name);
   return { status: 200, body: { paused: false } };
 }

package/src/status-render.ts

@@ -564,8 +564,19 @@ export function formatFleetStatus(
         })()
       : `dispatch  ${layers.dispatch}`;
 
+  const upgradeRecoveryBlock =
+    s.upgradeRecovery === undefined
+      ? undefined
+      : [
+          `upgrade   recovery required for failed omp-conductor@${s.upgradeRecovery.failedVersion} (${s.upgradeRecovery.phase})`,
+          `  journal  ${s.upgradeRecovery.journalPath}`,
+          `  owed     ${s.upgradeRecovery.verificationOwed}`,
+          "  release  automatic after successful verification; operator acknowledgement: `omp-conductor resume`",
+        ].join("\n");
+
   return [
     dispatchLine,
+    ...(upgradeRecoveryBlock === undefined ? [] : [upgradeRecoveryBlock]),
     tickLine,
     ...(nextTickLine === undefined ? [] : [nextTickLine]),
     paneLine,

package/src/upgrade-journal.ts

@@ -74,6 +74,8 @@ export interface UpgradeJournalEntry {
   pauseKey?: string;
   /** Whether dispatch was already paused when the request began. */
   initialPaused?: boolean;
+  /** Creation instant of the exact pause sentinel this transaction acquired. */
+  pauseSince?: number;
   /** The fleet layers the install began under, for delta verification. */
   initial?: JournalFleetState;
   /** The projects the install covers, in the form the upgrade engine uses. */

package/src/upgrade-verify.ts

@@ -25,7 +25,7 @@ import { readFileSync } from "node:fs";
 import { loadConfig, stateDir } from "./config.ts";
 import { DEFAULT_PORT } from "./lifecycle.ts";
 import { dbPath, openStore } from "./store.ts";
-import { appendJournal, readUpgradeJournal, upgradeJournalPath, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
+import { appendJournal, readUpgradeJournal, upgradeJournalPath, type JournalSurfaces, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
 import type { DoctorReport } from "./doctor.ts";
 import type { FleetLayers } from "./status-render.ts";
 import type { ReportDraft, ReportKind } from "./types.ts";
@@ -135,6 +135,17 @@ function newestRequestIndex(entries: readonly UpgradeJournalEntry[]): number {
   return -1;
 }
 
+/** Newest terminal outcome in one request's progression, without copying it. */
+function newestOutcome(
+  progression: readonly UpgradeJournalEntry[],
+): UpgradeJournalEntry | undefined {
+  for (let i = progression.length - 1; i >= 0; i--) {
+    const entry = progression[i]!;
+    if (entry.kind === "outcome") return entry;
+  }
+  return undefined;
+}
+
 /**
  * Classify one journal: no request, a request whose outcome is already
  * terminal, or a request still in flight — with the phase evidence that says
@@ -152,7 +163,7 @@ export function classifyUpgrade(
   const start = newestRequestIndex(entries);
   if (start === -1) return { kind: "none" };
   const progression = entries.slice(start);
-  const outcome = [...progression].reverse().find((entry) => entry.kind === "outcome");
+  const outcome = newestOutcome(progression);
   if (outcome !== undefined) {
     return { kind: "closed", phase: outcome.phase ?? "outcome" };
   }
@@ -167,15 +178,34 @@ export function classifyUpgrade(
   return { kind: "incomplete" };
 }
 
+/** Terminal outcomes that prove no broken or mixed install remains active. */
+const RESUMABLE_OUTCOMES = new Set([
+  "already-current",
+  "recovered",
+  "recovery-acknowledged",
+  "verified",
+]);
+/** Outcomes that require a durable host-wide safety fence. */
+const RECOVERY_HOLD_OUTCOMES = new Set([
+  "rollback-failed",
+  "rollback-requested",
+  "rollback-unavailable",
+  "rolled-back",
+]);
+/** Closed rollback outcomes whose installed surfaces must be re-verified. */
+const RECOVERY_CHECK_OUTCOMES = new Set(["rollback-failed", "rolled-back"]);
+
 /** One pending request plus the snapshot its verifier needs. */
 export interface PendingUpgradeRequest {
   version: string;
   gitHead?: string;
+  pauseSince?: number;
   initialPaused?: boolean;
   pauseKey?: string;
   selectors: readonly (string | undefined)[];
   initial?: UpgradeJournalEntry["initial"];
   configBackup?: string;
+  previous?: JournalSurfaces;
   /**
    * Epoch-ms when the install began (the journal snapshot's write time): the
    * deadline the live orchestrator session must have restarted after (#832).
@@ -202,6 +232,9 @@ export function pendingUpgradeRequest(
   const request = progression[0]!;
   if (request.ok === false) return undefined;
   const snapshot = progression.find((entry) => entry.kind === "snapshot");
+  const pausePhase = progression.find(
+    (entry) => entry.kind === "phase" && entry.phase === "paused",
+  );
   const snapshotAt = snapshot === undefined ? undefined : Date.parse(snapshot.at);
   return {
     version: request.version ?? "unknown",
@@ -209,12 +242,80 @@ export function pendingUpgradeRequest(
     initialPaused: snapshot?.initialPaused,
     pauseKey: snapshot?.pauseKey,
     selectors: (snapshot?.selectors ?? []).map((selector) => selector ?? undefined),
+    pauseSince: pausePhase?.pauseSince,
     initial: snapshot?.initial,
     configBackup: snapshot?.configBackup,
+    previous: snapshot?.previous,
     ...(snapshotAt !== undefined && Number.isFinite(snapshotAt) ? { reloadAfterMs: snapshotAt } : {}),
   };
 }
 
+export interface UpgradeRecoveryStatus {
+  failedVersion: string;
+  phase: string;
+  journalPath: string;
+  verificationOwed: string;
+}
+
+/**
+ * The unresolved safety hold represented by the newest journal request.
+ * Status reads this same lifecycle verdict as the verifier; it never guesses
+ * from the pause reason alone.
+ */
+export function upgradeRecoveryStatus(
+  entries: readonly UpgradeJournalEntry[],
+  root = stateDir(),
+): UpgradeRecoveryStatus | undefined {
+  const request = pendingUpgradeRequest(entries);
+  const state = classifyUpgrade(entries);
+  if (
+    request === undefined ||
+    state.kind !== "closed" ||
+    !RECOVERY_HOLD_OUTCOMES.has(state.phase)
+  ) {
+    return undefined;
+  }
+  const expected = request.previous;
+  const surfaces =
+    expected === undefined
+      ? "the pre-upgrade surface snapshot is missing"
+      : [
+          `CLI=${expected.cliVersion}`,
+          `omp=${expected.ompVersion ?? "absent"}`,
+          `herdr=${expected.herdrSource ?? "absent"}`,
+        ].join(", ");
+  const verificationOwed =
+    state.phase === "rollback-requested"
+      ? `detached rollback completion, then installed-surface verification against ${surfaces}`
+      : `installed-surface verification against ${surfaces}`;
+  return {
+    failedVersion: request.version,
+    phase: state.phase,
+    journalPath: upgradeJournalPath(root),
+    verificationOwed,
+  };
+}
+
+/**
+ * Record the operator's explicit choice to release an unresolved upgrade
+ * recovery fence. `resume` calls this before removing an upgrade-owned pause,
+ * so the append-only journal distinguishes acknowledgement from verification.
+ */
+export function acknowledgeUpgradeRecovery(root = stateDir()): boolean {
+  const entries = readUpgradeJournal(root);
+  const recovery = upgradeRecoveryStatus(entries, root);
+  if (recovery === undefined) return false;
+  appendJournal(root, {
+    at: new Date().toISOString(),
+    kind: "outcome",
+    phase: "recovery-acknowledged",
+    ok: true,
+    version: recovery.failedVersion,
+    detail: `operator resumed dispatch with ${recovery.verificationOwed} still owed`,
+  });
+  return true;
+}
+
 // ---------------------------------------------------------------------------
 // The live-orchestrator session attestation (#832)
 // ---------------------------------------------------------------------------
@@ -387,12 +488,18 @@ export interface UpgradeVerifyDeps {
   layers(project?: string): FleetLayers;
   health(port: number): Promise<{ ok: boolean; body?: string }>;
   doctor(projectName: string): Promise<DoctorReport>;
+  /** Read all three package/plugin identities from the live host. */
+  surfaces(): Promise<JournalSurfaces>;
   /** The durable outbox enqueue — one row, daemon-owned delivery. */
   enqueue(draft: ReportDraft): void;
   /** Read one pause sentinel (who set it, when) — daemon-owned state. */
   pause(project?: string): { source: string; reason?: string; since: number } | undefined;
   /** Lift the pause sentinel for a project, clearing the legacy global too. */
   resume(project?: string): void;
+  /** Escalate a project pause to the host-global safety fence. */
+  holdGlobal(): void;
+  /** Remove only the exact dedicated recovery fence observed by this pass. */
+  releaseGlobalRecovery(expectedSince: number | undefined): void;
   /** Start the detached rollback unit for a version. */
   launchRollback(
     version: string,
@@ -419,6 +526,8 @@ export interface UpgradeVerifyResult {
     | "none"
     | "already-closed"
     | "verified"
+    | "recovered"
+    | "recovery-owed"
     | "aborted"
     | "rollback-requested"
     | "rollback-unavailable";
@@ -453,41 +562,54 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
   if (request === undefined) return { handled: "none" };
   const state = classifyUpgrade(entries);
 
-  const journal = (entry: Omit<UpgradeJournalEntry, "at">): void => {
+  const journal = (entry: Omit<UpgradeJournalEntry, "at">, at = now()): void => {
     const write = deps.journal ?? ((line: UpgradeJournalEntry) => appendJournal(root, line));
-    write({ ...entry, version: entry.version ?? request.version, at: new Date(now()).toISOString() });
+    write({ ...entry, version: entry.version ?? request.version, at: new Date(at).toISOString() });
   };
 
   // ---------------------------------------------------------------- terminal
   if (state.kind === "closed") {
-    // The transaction is already terminal (verified, rolled back, aborted,
-    // rollback requested). One thing may still be owed: the install paused
-    // dispatch, and a crash took the process that should have resumed it. The
-    // resume is safe to redo — it only acts while the sentinel is still the
-    // install's own — so it runs on every visit to a closed request.
-    resumeUpgradePause(deps, request);
+    // A closed failure is still a safety hold: rollback-requested means the
+    // rollback unit has not landed yet, while rollback-unavailable and
+    // rollback-failed mean the host may contain mixed surfaces. Only outcomes
+    // that prove the transaction safe may release its pause.
+    const outcome = newestOutcome(progression);
+    const parsedOutcomeAt = outcome === undefined ? Number.NaN : Date.parse(outcome.at);
+    const outcomeAt = Number.isFinite(parsedOutcomeAt) ? parsedOutcomeAt : undefined;
+    if (RECOVERY_CHECK_OUTCOMES.has(state.phase)) {
+      return verifyRecoveredUpgrade(deps, request, state.phase, outcomeAt, journal, now);
+    }
+    if (RECOVERY_HOLD_OUTCOMES.has(state.phase)) deps.holdGlobal();
+    if (RESUMABLE_OUTCOMES.has(state.phase) && outcomeAt !== undefined) {
+      resumeUpgradePause(deps, request, outcomeAt);
+    }
     return { handled: "already-closed", version: request.version, detail: state.phase };
   }
 
   // ------------------------------------------------------------ never started
   if (state.kind === "no-start") {
-    // The unit died before journaling a single phase. Nothing was installed;
-    // the sentinel can still exist if it crashed between pausing and writing
-    // the pause line — clear it, close the request, and page tier 2 that the
-    // fleet could not install its own fix.
-    resumeUpgradePause(deps, request);
+    // The unit died before journaling a single phase. Nothing was installed,
+    // but no phase recorded the exact pause instance either. Close the request
+    // without guessing that an upgrade-owned sentinel still belongs to it.
+    const abortedAt = now();
     deps.enqueue({
       project: deps.projectName,
       kind: "tier2",
       body: [
         `Fleet upgrade to omp-conductor@${request.version} never started (journal: ${upgradeJournalPath(root)}).`,
-        "The detached install unit died before touching any surface; dispatch was not paused.",
+        "The detached install unit died before touching any surface.",
+        "Dispatch may still hold an unidentifiable install pause; check `status` and clear it explicitly with `omp-conductor resume`.",
         `Re-run the request, or install by hand with \`omp-conductor upgrade --to ${request.version}\`.`,
       ].join("\n"),
-      at: now(),
+      at: abortedAt,
       dedupeKey: `upgrade:${request.version}:aborted`,
     });
-    journal({ kind: "outcome", phase: "aborted", ok: false, detail: "requested upgrade never started a phase" });
+    journal({
+      kind: "outcome",
+      phase: "aborted",
+      ok: false,
+      detail: "requested upgrade never started a phase",
+    }, abortedAt);
     return { handled: "aborted", version: request.version, detail: "no phase was ever journaled" };
   }
 
@@ -518,16 +640,9 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
     );
   }
 
-  // Good. Resume dispatch only when the install paused it, and only while the
-  // sentinel is still the install's own — an operator's later hold is never
-  // lifted by an upgrade closing.
-  if (request.initialPaused === false) {
-    const owned = deps.pause(request.pauseKey);
-    if (owned !== undefined && owned.source === "upgrade") {
-      deps.resume(request.pauseKey);
-      deps.log(`upgrade verified — dispatch resumed (${request.pauseKey ?? "global"})`);
-    }
-  }
+  // Good. Persist the evidence before releasing anything: if this process dies
+  // after the outcome, the next tick can finish the idempotent resume.
+  const verifiedAt = now();
   deps.enqueue({
     project: deps.projectName,
     kind: "material",
@@ -538,23 +653,118 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
       "",
       "Dispatch restored to its prior state.",
     ].join("\n"),
-    at: now(),
+    at: verifiedAt,
     dedupeKey: `upgrade:${request.version}:verified`,
   });
-  journal({ kind: "outcome", phase: "verified", ok: true, checks });
+  journal({ kind: "outcome", phase: "verified", ok: true, checks }, verifiedAt);
+  const paused = deps.pause(request.pauseKey);
+  resumeUpgradePause(deps, request, verifiedAt);
+  if (paused !== undefined && deps.pause(request.pauseKey) === undefined) {
+    deps.log(`upgrade verified — dispatch resumed (${request.pauseKey ?? "global"})`);
+  }
   return { handled: "verified", version: request.version };
 }
 
+async function verifyRecoveredUpgrade(
+  deps: UpgradeVerifyDeps,
+  request: PendingUpgradeRequest,
+  failedPhase: string,
+  failedOutcomeAt: number | undefined,
+  journal: (entry: Omit<UpgradeJournalEntry, "at">, at?: number) => void,
+  now: () => number,
+): Promise<UpgradeVerifyResult> {
+  const globalPause = deps.pause();
+  const globalRecoverySince =
+    globalPause?.source === "upgrade-recovery" ? globalPause.since : undefined;
+  const checks: UpgradeCheck[] = [];
+  const expected = request.previous;
+  if (expected === undefined) {
+    checks.push({ name: "installed-surfaces", ok: false, detail: "pre-upgrade snapshot is missing" });
+  } else {
+    try {
+      const actual = await deps.surfaces();
+      for (const [name, wanted, found] of [
+        ["cli", expected.cliVersion, actual.cliVersion],
+        ["omp", expected.ompVersion, actual.ompVersion],
+        ["herdr", expected.herdrSource, actual.herdrSource],
+      ] as const) {
+        checks.push({
+          name: `surface:${name}`,
+          ok: wanted === found,
+          detail: `expected ${wanted ?? "absent"}; found ${found ?? "absent"}`,
+        });
+      }
+    } catch (err) {
+      checks.push({
+        name: "installed-surfaces",
+        ok: false,
+        detail: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+  const failed = checks.find((check) => !check.ok);
+  if (failed !== undefined) {
+    // A recovery verdict owns the host-wide gate. Re-establish it on every
+    // owed pass so a legacy failed journal or a lost sentinel cannot admit.
+    deps.holdGlobal();
+    return {
+      handled: "recovery-owed",
+      version: request.version,
+      detail: `${failedPhase}; ${failed.name}: ${failed.detail ?? "failed"}`,
+    };
+  }
+
+  const recoveredAt = now();
+  deps.enqueue({
+    project: deps.projectName,
+    kind: "material",
+    body: [
+      `The failed upgrade to omp-conductor@${request.version} is recovered.`,
+      "",
+      ...checks.map((check) => `ok  ${check.name} — ${check.detail ?? "matched"}`),
+      "",
+      "Every installed surface matches the durable pre-upgrade snapshot; dispatch restored to its prior state.",
+    ].join("\n"),
+    at: recoveredAt,
+    dedupeKey: `upgrade:${request.version}:recovered`,
+  });
+  journal({
+    kind: "outcome",
+    phase: "recovered",
+    ok: true,
+    detail: `verified recovery after ${failedPhase}`,
+    checks,
+  }, recoveredAt);
+  resumeUpgradePause(deps, request, failedOutcomeAt);
+  deps.releaseGlobalRecovery(globalRecoverySince);
+  return { handled: "recovered", version: request.version, detail: failedPhase };
+}
+
 /**
  * Lift the install's own pause sentinel, and only that one. `pauseKey` is the
  * sentinel the engine paused; a fixed `initialPaused` true means the fleet was
  * already held and the install never paused it, so nothing is lifted. A pause
  * that an operator re-created (source changed) is left standing.
+ *
+ * New journals carry the exact pause creation instant, so identical
+ * provenance can never make a newer transaction look owned. `notAfter` is the
+ * compatibility bound for older journals that predate that identity field:
+ * only a pause that existed before the safe terminal outcome may be released.
  */
-function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequest): void {
+function resumeUpgradePause(
+  deps: UpgradeVerifyDeps,
+  request: PendingUpgradeRequest,
+  notAfter?: number,
+): void {
   if (request.initialPaused === true) return;
   const owned = deps.pause(request.pauseKey);
-  if (owned !== undefined && owned.source === "upgrade") {
+  const sameTransaction =
+    request.pauseSince !== undefined
+      ? owned !== undefined &&
+        owned.since >= request.pauseSince &&
+        (notAfter === undefined || owned.since <= notAfter)
+      : notAfter !== undefined && owned !== undefined && owned.since <= notAfter;
+  if (owned?.source === "upgrade" && sameTransaction) {
     deps.resume(request.pauseKey);
   }
 }
@@ -687,6 +897,10 @@ async function triggerUpgradeRollback(
   at: number,
   why: string,
 ): Promise<UpgradeVerifyResult> {
+  // Every install surface is host-wide even when the request named one
+  // project. Once recovery is required, the safety fence must gate every
+  // project served by the shared daemon.
+  deps.holdGlobal();
   // The marker is the spawn guard: present-and-ok means a rollback unit is
   // already on its way and a crash after the spawn must not spawn a second
   // one. Present-and-failed means nothing started, so a retry is a first

package/src/upgrade.ts

@@ -129,6 +129,8 @@ export interface UpgradeDeps {
    */
   pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
   setPaused(value: boolean, project?: string): void;
+  /** Host-global recovery fence for host-wide install surfaces. */
+  setGlobalRecoveryPaused(value: boolean): void;
   restartDaemon(): Promise<void>;
   sleep(ms: number): Promise<void>;
   /**
@@ -205,6 +207,14 @@ export const DEFAULT_DEPS: UpgradeDeps = {
   pauseState: (project) => pauseInstance(project),
   setPaused: (v, project) =>
     setPaused(v, { source: "upgrade", reason: "upgrade, draining" }, project),
+  setGlobalRecoveryPaused: (v) => {
+    const current = pauseInstance();
+    if (v && current === undefined) {
+      setPaused(true, { source: "upgrade-recovery", reason: "upgrade recovery verification required" });
+    } else if (!v && current?.source === "upgrade-recovery") {
+      setPaused(false);
+    }
+  },
   restartDaemon: async () => {
     await restartDaemon({});
   },
@@ -1413,7 +1423,14 @@ export async function upgradeConductor(
     deps.log("safety: pausing new issue claims");
     deps.setPaused(true, scope.pauseKey);
   }
-  journal({ kind: "phase", phase: "paused", surface: "dispatch", ok: true, version: release.version });
+  journal({
+    kind: "phase",
+    phase: "paused",
+    surface: "dispatch",
+    ok: true,
+    version: release.version,
+    pauseSince: deps.pauseState(scope.pauseKey)?.since,
+  });
   deps.log("drain: waiting for live omp worker sessions");
   try {
     await waitForDrain(deps, scope);
@@ -1595,6 +1612,7 @@ export async function upgradeConductor(
     } catch {
       deps.setPaused(true, scope.pauseKey);
     }
+    deps.setGlobalRecoveryPaused(true);
     const failure = err instanceof Error ? err.message : String(err);
     journal({
       kind: "phase",
@@ -1779,6 +1797,10 @@ export async function rollbackFromJournal(
     configBefore,
     request.configBackup,
   );
+  // The restored bytes remain host-wide unverified state until the returning
+  // daemon reads all three surfaces. A project-scoped request cannot leave its
+  // siblings admitting in that interval.
+  deps.setGlobalRecoveryPaused(true);
 
   const restoredVersion = previous.cliVersion;
   appendJournal(root, {