omp-conductor 0.19.3 → 0.19.5

package/README.md

@@ -386,9 +386,10 @@ message on stderr) on every uncertainty:
 - the tick config does not parse — the agent name would be a guess
 - `herdr agent list` is unreachable, prints nothing, or prints output with no
   explicit `agents` array; only a real `agents: []` means "no agents"
-- an agent row is unreadable — a missing `name`/`pane_id`, or an `agent` field
-  present with a non-string value. An *absent* or `null` `agent` is the sticky
-  claim herdr reports after the agent exits, and stays a normal answer
+- an agent row is unreadable — a missing `pane_id`, a non-string non-null
+  `name`, or an `agent` field present with a non-string value. An absent or
+  `null` `name` is an unnamed pane; an absent or `null` `agent` is the sticky
+  claim herdr reports after the agent exits, and both stay normal answers
 - the configured agent name is not unique, or the claimed pane runs some other agent
 - `pane process-info` fails, or the claim is live `omp` but names no recognizable
   omp foreground PID — "cannot see it" is never reported as "it is stopped"

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.19.3",
+  "version": "0.19.5",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/src/commands/resume.ts

@@ -13,10 +13,15 @@
 
 import type { CommandContext } from "./context.ts";
 import { armState, clearPaneHaltIfResolvable, releaseHold } from "../fleet.ts";
+import { acknowledgeUpgradeRecovery } from "../upgrade-verify.ts";
 import { wakeDispatch } from "../wake.ts";
 
 export async function resumeCommand(ctx: CommandContext): Promise<void> {
 for (const project of ctx.targetProjects()) {
+  // `releaseHold(project)` also removes the host-global sentinel. Record an
+  // unresolved recovery acknowledgement first even when a project-local pause
+  // shadows that global fence; the helper is a no-op when nothing is owed.
+  acknowledgeUpgradeRecovery();
   releaseHold(project.name);
   const pin = clearPaneHaltIfResolvable(project.name);
   const arm = armState(project.name);

package/src/daemon.ts

@@ -58,7 +58,7 @@ import {
   setPaused,
 } from "./pause.ts";
 import { appendJournal, launchTransientUnit, readUpgradeJournal } from "./upgrade-journal.ts";
-import { runCommand, verifyPendingUpgrade, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
+import { runCommand, upgradeRecoveryStatus, verifyPendingUpgrade, type UpgradeRecoveryStatus, type UpgradeVerifyDeps } from "./upgrade-verify.ts";
 import { inspectSurfaces, type InstalledSurfaces } from "./upgrade.ts";
 import { checkTelegramFreshness, type TelegramFreshness } from "./telegram-freshness.ts";
 import { processStartTimeMs } from "./upgrade-verify.ts";
@@ -116,11 +116,13 @@ import {
 import {
   adoptSalvagedPrs,
   classifyAndRecover,
+  harnessLogDir,
   collectSettlementFlags,
   formatQuarantinedRuns,
   formatSalvagedRuns,
   reactToProviderCredit,
   readSessionError,
+  readTerminalEvidence,
   reconcileOrphanedRuns,
   reconcileGroomingClosures,
   reconcileStaleLabels,
@@ -133,6 +135,7 @@ import { materializeOmpSettings } from "./omp-settings.ts";
 import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
 import {
   infraLogSignature,
+  classifyRun,
   infraSignatureVersion,
   providerCreditRefusal,
   providerTransientFault,
@@ -333,6 +336,8 @@ interface Deps {
   workerControls: WorkerControlRegistry;
   /** Session seam for lifecycle integration tests; production uses the real harness. */
   workerDeps?: RunWorkerDeps;
+  /** Harness per-process logs used to classify terminal worker routes. */
+  harnessLogDir?: string;
   integrity: IntegrityGate;
   stall: StallGate;
   /**
@@ -3959,7 +3964,10 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
     if (await settleStopBeforeSession()) return;
     if (await settleDrainBeforeSession()) return;
 
-    store.updateRun(runId, { worktree: worktreePath });
+    // A review revision is a new worker process on the same run row. Clear the
+    // prior process evidence before launch so a failed wake cannot inherit an
+    // earlier round's harness route.
+    store.updateRun(runId, { worktree: worktreePath, workerPid: null, terminalEvidence: null });
 
     // The row's own counters are cumulative across the revision: the revision
     // worker meters its own session from zero, so its deltas are added to the
@@ -3972,6 +3980,7 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
     log(`#${issue} review round ${revision.round} → resuming ${branch} (${priorSessionFile})`);
 
 
+    let workerStartedAt: number | undefined;
     let result: WorkerResult;
     try {
       const runAllowanceUsd = runSpendAllowanceUsd(caps);
@@ -3997,7 +4006,9 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
         socketPath: join(sessionDir, "ipc.sock"),
         verbSocketPath: verbListener.path,
         onSpawn: (pid) => {
+          workerStartedAt = Date.now();
           verbListener?.bindPid(pid);
+          store.updateRun(runId, { workerPid: pid });
         },
         onChildLog: (line) => {
           log(`#${issue} ${line}`);
@@ -4092,6 +4103,18 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
       ...(verified.reason === undefined ? [] : ["", verified.reason]),
       result.report,
     ].join("\n");
+    const latestRun = store.getRun(runId) ?? run;
+    const terminalEvidence =
+      state === "failed" || state === "killed"
+        ? readTerminalEvidence(
+            {
+              sessionFile: result.sessionFile,
+              startedAt: workerStartedAt ?? latestRun.startedAt,
+              ...(latestRun.workerPid === undefined ? {} : { workerPid: latestRun.workerPid }),
+            },
+            d.harnessLogDir ?? harnessLogDir(),
+          )
+        : undefined;
 
     const terminalPatch: Partial<RunRecord> = {
       endedAt: Date.now(),
@@ -4112,9 +4135,76 @@ export async function handleReviewRevision(d: Deps, revision: ReviewRevisionReco
       ...(result.headSha === undefined ? {} : { headSha: result.headSha }),
       sessionFile: result.sessionFile,
       ...(result.graphTools === undefined ? {} : { graphTools: result.graphTools }),
+      ...(terminalEvidence === undefined ? {} : { terminalEvidence }),
       report: finalReport,
       ...settlement?.patch,
     };
+    const terminalClassification =
+      terminalEvidence === undefined
+        ? undefined
+        : classifyRun(
+            {
+              ...latestRun,
+              ...terminalPatch,
+              state,
+              terminalEvidence,
+            },
+            {},
+            caps,
+          );
+    if (terminalClassification?.cls === "model-empty-stop") {
+      const recoveredAt = Date.now();
+      store.updateRun(runId, {
+        ...terminalPatch,
+        state: "failed",
+        failureClass: terminalClassification.cls,
+        recoveryAction: terminalClassification.recovery,
+        recoveredAt,
+      });
+      const continuations = store.continuationsFor(project.name, issue);
+      if (hasContinuationBudget(continuations, caps.maxContinuationsPerIssue)) {
+        // Keep the same review row, round, findings, session, branch and PR.
+        // Restore the exact pushed-green origin state while moving this stop
+        // into the explicit continuation counter; the next claim must not
+        // depend on re-verifying a failed-state origin at an unchanged head.
+        const chargedRun = store.getRun(runId) ?? latestRun;
+        store.updateRun(runId, {
+          ...terminalPatch,
+          state: "pushed-green",
+          failureClass: null,
+          recoveryAction: null,
+          recoveredAt: null,
+          continuationCharges: (chargedRun.continuationCharges ?? 0) + 1,
+          workerPid: null,
+          lastError: terminalClassification.evidence,
+        });
+        store.requeueReviewRevision(revision.id);
+        log(
+          `#${issue} review round ${revision.round} re-queued after a provider empty-stop ` +
+            `(${continuations}/${caps.maxContinuationsPerIssue} continuations): ${terminalClassification.evidence}`,
+        );
+        return;
+      }
+
+      store.settleReviewRevision(revision.id, "failed", recoveredAt);
+      swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
+      const model = result.model ?? latestRun.resolvedModel ?? latestRun.model ?? "configured model";
+      const provider = result.provider ?? latestRun.resolvedProvider ?? "provider";
+      await safeEscalate(d, {
+        tier: 1,
+        project: project.name,
+        issue,
+        runId,
+        summary: `#${issue} review round ${revision.round}: the model kept returning empty responses`,
+        detail:
+          `${provider}/${model} ended ${continuations} review session(s) with empty responses until ` +
+          `the harness retry cap. The same review round was preserved while continuation budget remained; ` +
+          `that budget is now exhausted at ${caps.maxContinuationsPerIssue}. No implementation failure or ` +
+          `additional review round was charged.\n\n${terminalClassification.evidence}`,
+      });
+      log(`#${issue} review round ${revision.round} provider empty-stops exhausted the continuation budget`);
+      return;
+    }
     // #903: an infrastructure kill spends no review round.
     //
     // A dispatch that dies before the resumed session takes a turn — a host
@@ -5205,6 +5295,7 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
     layers: (project) => fleetLayers(project),
     health: healthCheck,
     doctor: (name) => runDoctor(name),
+    surfaces: () => inspectSurfaces({ run: runCommand, log, env: process.env }),
     enqueue: (draft) => {
       try {
         d.store.enqueueReport(draft);
@@ -5214,6 +5305,21 @@ export function upgradeVerifyDepsFor(d: Pick<Deps, "project" | "store">): Upgrad
     },
     pause: pauseInstance,
     resume: (project) => setPaused(false, undefined, project),
+    holdGlobal: () => {
+      if (pauseInstance() === undefined) {
+        setPaused(true, { source: "upgrade-recovery", reason: "upgrade recovery verification required" });
+      }
+    },
+    releaseGlobalRecovery: (expectedSince) => {
+      const pause = pauseInstance();
+      if (
+        pause?.source === "upgrade-recovery" &&
+        expectedSince !== undefined &&
+        pause.since === expectedSince
+      ) {
+        setPaused(false);
+      }
+    },
     launchRollback: async (version) => {
       const launched = await launchTransientUnit(
         (argv) => runCommand(argv[0]!, argv.slice(1)),
@@ -5243,7 +5349,11 @@ async function verifyPendingUpgradeTick(
   const verifyDeps = upgradeVerifyDepsFor(d);
   try {
     const outcome = await run(verifyDeps);
-    if (outcome.handled !== "none" && outcome.handled !== "already-closed") {
+    if (
+      outcome.handled !== "none" &&
+      outcome.handled !== "already-closed" &&
+      outcome.handled !== "recovery-owed"
+    ) {
       log(
         `upgrade journal handled: ${outcome.handled}` +
           `${outcome.version === undefined ? "" : ` for omp-conductor@${outcome.version}`}` +
@@ -6390,6 +6500,8 @@ export interface StatusSnapshot {
    * reading like a mistake (#220).
    */
   pauseReason?: string;
+  /** Unresolved host-wide upgrade recovery, rendered identically for every project. */
+  upgradeRecovery?: UpgradeRecoveryStatus;
   /**
    * The project's active self-expiring drain (#484): present only while a
    * fresh, valid drain record exists. Paused and drained are deliberately two
@@ -6584,9 +6696,12 @@ export function statusSnapshotFromStore(
   const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
   // Read once: `status` renders the degrade row off this while it is down.
   const orchestratorDown = store.orchestratorIncident(p.name);
-  // Read once: the provenance read touches the filesystem, and the renderer
-  // should never pay for it twice per status.
-  const reason = pauseProvenance(p.name)?.reason;
+  // The journal is the authoritative recovery lifecycle. Render unresolved
+  // recovery even when an operator/project pause currently wins provenance,
+  // or before the verifier has re-created a missing global fence.
+  const pause = pauseProvenance(p.name);
+  const reason = pause?.reason;
+  const upgradeRecovery = upgradeRecoveryStatus(readUpgradeJournal(stateDir()), stateDir());
   // Same cost discipline as the pause read: the drain record is a file read,
   // and the active-drain view is only built when one is actually fresh. The
   // read is observational — it never mutates the record — so a status read
@@ -6651,6 +6766,7 @@ export function statusSnapshotFromStore(
     stateDir: stateDir(),
     paused: isPaused(p.name),
     ...(reason === undefined ? {} : { pauseReason: reason }),
+    ...(upgradeRecovery === undefined ? {} : { upgradeRecovery }),
     ...(drain.kind === "active"
       ? {
           drain: {

package/src/dashboard/controls.ts

@@ -46,6 +46,7 @@ import { unblockIssue } from "../unblock.ts";
 import { makeTracker } from "../tracker/github.ts";
 import { dbPath, openStore } from "../store.ts";
 import { loadConfig, resolveCaps } from "../config.ts";
+import { acknowledgeUpgradeRecovery } from "../upgrade-verify.ts";
 import type { ProjectConfig } from "../types.ts";
 
 /** The source string every dashboard mutation attributes itself with. */
@@ -70,6 +71,7 @@ export interface ControlDeps {
   project: ProjectConfig;
   hold?: typeof hold;
   releaseHold?: typeof releaseHold;
+  acknowledgeUpgradeRecovery?: typeof acknowledgeUpgradeRecovery;
   disarmTicks?: typeof disarmTicks;
   /**
    * Forwards one request to the owning daemon's own HTTP surface. Returns
@@ -119,6 +121,7 @@ export function dashboardPause(body: unknown, d: ControlDeps): ControlOutcome {
 
 /** Clear the pause. Idempotent, exactly as `resume` is on the CLI. */
 export function dashboardResume(d: ControlDeps): ControlOutcome {
+  (d.acknowledgeUpgradeRecovery ?? acknowledgeUpgradeRecovery)();
   (d.releaseHold ?? releaseHold)(d.project.name);
   return { status: 200, body: { paused: false } };
 }

package/src/fleet.ts

@@ -815,7 +815,7 @@ export function startHerdrFleet(projectName?: string, deps: HerdrStartDeps = {})
 }
 
 export interface HerdrAgent {
-  name: string;
+  name?: string;
   paneId: string;
   agent?: string;
   sessionPath?: string;
@@ -1568,16 +1568,22 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
     const agent = row as { readonly [key: string]: unknown };
     const name = agent["name"];
     const paneId = agent["pane_id"];
-    if (typeof name !== "string" || typeof paneId !== "string") {
+    if (typeof paneId !== "string") {
+      throw new Error(
+        "herdr agent list row is missing a string `pane_id` — cannot identify the pane",
+      );
+    }
+    if (name !== undefined && name !== null && typeof name !== "string") {
       throw new Error(
-        "herdr agent list row is missing a string `name`/`pane_id` — " +
+        `herdr agent list row for pane ${paneId} has a non-string \`name\` (${typeof name}) — ` +
           "cannot tell whether it is the conductor pane",
       );
     }
+    const parsedName = typeof name === "string" ? name : undefined;
     const rawAgent = agent["agent"];
     if (rawAgent !== undefined && rawAgent !== null && typeof rawAgent !== "string") {
       throw new Error(
-        `herdr agent list row for ${name} has a non-string \`agent\` (${typeof rawAgent}) — ` +
+        `herdr agent list row for ${parsedName ?? `pane ${paneId}`} has a non-string \`agent\` (${typeof rawAgent}) — ` +
           `cannot tell whether an agent is live`,
       );
     }
@@ -1591,7 +1597,7 @@ export function parseHerdrAgentList(rawOutput: string): HerdrAgent[] {
       }
     }
     out.push({
-      name,
+      ...(parsedName === undefined ? {} : { name: parsedName }),
       paneId,
       ...(liveAgent === undefined ? {} : { agent: liveAgent }),
       ...(sessionPath === undefined ? {} : { sessionPath }),

package/src/settlement.ts

@@ -1287,10 +1287,10 @@ export function harnessLogDir(): string {
  *  per process. These name why a session ended when the row cannot: the run
  *  itself records no error, because from its side nothing failed (#986). */
 const HARNESS_ROUTE_MARKERS = [
-  "empty-stop-retry-cap",
-  "empty-stop-handled",
-  "bottom-checkCompaction",
-  "context-overflow",
+  { marker: "empty-stop-retry-cap", terminal: true },
+  { marker: "empty-stop-handled", terminal: false },
+  { marker: "bottom-checkCompaction", terminal: false },
+  { marker: "context-overflow", terminal: true },
 ] as const;
 
 /**
@@ -1373,13 +1373,27 @@ function readHarnessRoute(
   } catch {
     return undefined;
   }
-  // Last match wins: the terminal route is the one that ended the session, and
-  // `empty-stop-handled` appearing earlier is a retry that worked.
+  // A terminal failure route remains the cause even when the harness performs
+  // post-agent maintenance afterwards. The live #1001 logs record
+  // `empty-stop-retry-cap` and then `bottom-checkCompaction` milliseconds later;
+  // choosing the last marker hid the failure behind routine bookkeeping.
   let found: string | undefined;
-  for (const marker of HARNESS_ROUTE_MARKERS) {
-    const at = text.lastIndexOf(marker);
-    if (at === -1) continue;
-    if (found === undefined || at > text.lastIndexOf(found)) found = marker;
+  let foundAt = -1;
+  for (const route of HARNESS_ROUTE_MARKERS) {
+    if (!route.terminal) continue;
+    const at = text.lastIndexOf(route.marker);
+    if (at > foundAt) {
+      found = route.marker;
+      foundAt = at;
+    }
+  }
+  if (found !== undefined) return `harness route ${found}`;
+  for (const route of HARNESS_ROUTE_MARKERS) {
+    const at = text.lastIndexOf(route.marker);
+    if (at > foundAt) {
+      found = route.marker;
+      foundAt = at;
+    }
   }
   return found === undefined ? undefined : `harness route ${found}`;
 }

package/src/status-render.ts

@@ -564,8 +564,19 @@ export function formatFleetStatus(
         })()
       : `dispatch  ${layers.dispatch}`;
 
+  const upgradeRecoveryBlock =
+    s.upgradeRecovery === undefined
+      ? undefined
+      : [
+          `upgrade   recovery required for failed omp-conductor@${s.upgradeRecovery.failedVersion} (${s.upgradeRecovery.phase})`,
+          `  journal  ${s.upgradeRecovery.journalPath}`,
+          `  owed     ${s.upgradeRecovery.verificationOwed}`,
+          "  release  automatic after successful verification; operator acknowledgement: `omp-conductor resume`",
+        ].join("\n");
+
   return [
     dispatchLine,
+    ...(upgradeRecoveryBlock === undefined ? [] : [upgradeRecoveryBlock]),
     tickLine,
     ...(nextTickLine === undefined ? [] : [nextTickLine]),
     paneLine,

package/src/store.ts

@@ -3758,7 +3758,7 @@ export function openStore(dbPath: string): Store {
          ELSE 0
        END,
        continuationCharges = COALESCE(continuationCharges, 0) + CASE
-         WHEN state = 'failed' AND failureClass = 'returned-for-revision' THEN 1
+         WHEN state = 'failed' AND failureClass IN ('returned-for-revision', 'model-empty-stop') THEN 1
          WHEN state = 'killed' AND (failureClass IS NULL OR failureClass NOT IN (${CHARGEABLE_CONTINUATION_EXCLUSIONS})) THEN 1
          ELSE 0
        END

package/src/upgrade-journal.ts

@@ -74,6 +74,8 @@ export interface UpgradeJournalEntry {
   pauseKey?: string;
   /** Whether dispatch was already paused when the request began. */
   initialPaused?: boolean;
+  /** Creation instant of the exact pause sentinel this transaction acquired. */
+  pauseSince?: number;
   /** The fleet layers the install began under, for delta verification. */
   initial?: JournalFleetState;
   /** The projects the install covers, in the form the upgrade engine uses. */

package/src/upgrade-verify.ts

@@ -25,7 +25,7 @@ import { readFileSync } from "node:fs";
 import { loadConfig, stateDir } from "./config.ts";
 import { DEFAULT_PORT } from "./lifecycle.ts";
 import { dbPath, openStore } from "./store.ts";
-import { appendJournal, readUpgradeJournal, upgradeJournalPath, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
+import { appendJournal, readUpgradeJournal, upgradeJournalPath, type JournalSurfaces, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
 import type { DoctorReport } from "./doctor.ts";
 import type { FleetLayers } from "./status-render.ts";
 import type { ReportDraft, ReportKind } from "./types.ts";
@@ -135,6 +135,17 @@ function newestRequestIndex(entries: readonly UpgradeJournalEntry[]): number {
   return -1;
 }
 
+/** Newest terminal outcome in one request's progression, without copying it. */
+function newestOutcome(
+  progression: readonly UpgradeJournalEntry[],
+): UpgradeJournalEntry | undefined {
+  for (let i = progression.length - 1; i >= 0; i--) {
+    const entry = progression[i]!;
+    if (entry.kind === "outcome") return entry;
+  }
+  return undefined;
+}
+
 /**
  * Classify one journal: no request, a request whose outcome is already
  * terminal, or a request still in flight — with the phase evidence that says
@@ -152,7 +163,7 @@ export function classifyUpgrade(
   const start = newestRequestIndex(entries);
   if (start === -1) return { kind: "none" };
   const progression = entries.slice(start);
-  const outcome = [...progression].reverse().find((entry) => entry.kind === "outcome");
+  const outcome = newestOutcome(progression);
   if (outcome !== undefined) {
     return { kind: "closed", phase: outcome.phase ?? "outcome" };
   }
@@ -167,15 +178,34 @@ export function classifyUpgrade(
   return { kind: "incomplete" };
 }
 
+/** Terminal outcomes that prove no broken or mixed install remains active. */
+const RESUMABLE_OUTCOMES = new Set([
+  "already-current",
+  "recovered",
+  "recovery-acknowledged",
+  "verified",
+]);
+/** Outcomes that require a durable host-wide safety fence. */
+const RECOVERY_HOLD_OUTCOMES = new Set([
+  "rollback-failed",
+  "rollback-requested",
+  "rollback-unavailable",
+  "rolled-back",
+]);
+/** Closed rollback outcomes whose installed surfaces must be re-verified. */
+const RECOVERY_CHECK_OUTCOMES = new Set(["rollback-failed", "rolled-back"]);
+
 /** One pending request plus the snapshot its verifier needs. */
 export interface PendingUpgradeRequest {
   version: string;
   gitHead?: string;
+  pauseSince?: number;
   initialPaused?: boolean;
   pauseKey?: string;
   selectors: readonly (string | undefined)[];
   initial?: UpgradeJournalEntry["initial"];
   configBackup?: string;
+  previous?: JournalSurfaces;
   /**
    * Epoch-ms when the install began (the journal snapshot's write time): the
    * deadline the live orchestrator session must have restarted after (#832).
@@ -202,6 +232,9 @@ export function pendingUpgradeRequest(
   const request = progression[0]!;
   if (request.ok === false) return undefined;
   const snapshot = progression.find((entry) => entry.kind === "snapshot");
+  const pausePhase = progression.find(
+    (entry) => entry.kind === "phase" && entry.phase === "paused",
+  );
   const snapshotAt = snapshot === undefined ? undefined : Date.parse(snapshot.at);
   return {
     version: request.version ?? "unknown",
@@ -209,12 +242,80 @@ export function pendingUpgradeRequest(
     initialPaused: snapshot?.initialPaused,
     pauseKey: snapshot?.pauseKey,
     selectors: (snapshot?.selectors ?? []).map((selector) => selector ?? undefined),
+    pauseSince: pausePhase?.pauseSince,
     initial: snapshot?.initial,
     configBackup: snapshot?.configBackup,
+    previous: snapshot?.previous,
     ...(snapshotAt !== undefined && Number.isFinite(snapshotAt) ? { reloadAfterMs: snapshotAt } : {}),
   };
 }
 
+export interface UpgradeRecoveryStatus {
+  failedVersion: string;
+  phase: string;
+  journalPath: string;
+  verificationOwed: string;
+}
+
+/**
+ * The unresolved safety hold represented by the newest journal request.
+ * Status reads this same lifecycle verdict as the verifier; it never guesses
+ * from the pause reason alone.
+ */
+export function upgradeRecoveryStatus(
+  entries: readonly UpgradeJournalEntry[],
+  root = stateDir(),
+): UpgradeRecoveryStatus | undefined {
+  const request = pendingUpgradeRequest(entries);
+  const state = classifyUpgrade(entries);
+  if (
+    request === undefined ||
+    state.kind !== "closed" ||
+    !RECOVERY_HOLD_OUTCOMES.has(state.phase)
+  ) {
+    return undefined;
+  }
+  const expected = request.previous;
+  const surfaces =
+    expected === undefined
+      ? "the pre-upgrade surface snapshot is missing"
+      : [
+          `CLI=${expected.cliVersion}`,
+          `omp=${expected.ompVersion ?? "absent"}`,
+          `herdr=${expected.herdrSource ?? "absent"}`,
+        ].join(", ");
+  const verificationOwed =
+    state.phase === "rollback-requested"
+      ? `detached rollback completion, then installed-surface verification against ${surfaces}`
+      : `installed-surface verification against ${surfaces}`;
+  return {
+    failedVersion: request.version,
+    phase: state.phase,
+    journalPath: upgradeJournalPath(root),
+    verificationOwed,
+  };
+}
+
+/**
+ * Record the operator's explicit choice to release an unresolved upgrade
+ * recovery fence. `resume` calls this before removing an upgrade-owned pause,
+ * so the append-only journal distinguishes acknowledgement from verification.
+ */
+export function acknowledgeUpgradeRecovery(root = stateDir()): boolean {
+  const entries = readUpgradeJournal(root);
+  const recovery = upgradeRecoveryStatus(entries, root);
+  if (recovery === undefined) return false;
+  appendJournal(root, {
+    at: new Date().toISOString(),
+    kind: "outcome",
+    phase: "recovery-acknowledged",
+    ok: true,
+    version: recovery.failedVersion,
+    detail: `operator resumed dispatch with ${recovery.verificationOwed} still owed`,
+  });
+  return true;
+}
+
 // ---------------------------------------------------------------------------
 // The live-orchestrator session attestation (#832)
 // ---------------------------------------------------------------------------
@@ -387,12 +488,18 @@ export interface UpgradeVerifyDeps {
   layers(project?: string): FleetLayers;
   health(port: number): Promise<{ ok: boolean; body?: string }>;
   doctor(projectName: string): Promise<DoctorReport>;
+  /** Read all three package/plugin identities from the live host. */
+  surfaces(): Promise<JournalSurfaces>;
   /** The durable outbox enqueue — one row, daemon-owned delivery. */
   enqueue(draft: ReportDraft): void;
   /** Read one pause sentinel (who set it, when) — daemon-owned state. */
   pause(project?: string): { source: string; reason?: string; since: number } | undefined;
   /** Lift the pause sentinel for a project, clearing the legacy global too. */
   resume(project?: string): void;
+  /** Escalate a project pause to the host-global safety fence. */
+  holdGlobal(): void;
+  /** Remove only the exact dedicated recovery fence observed by this pass. */
+  releaseGlobalRecovery(expectedSince: number | undefined): void;
   /** Start the detached rollback unit for a version. */
   launchRollback(
     version: string,
@@ -419,6 +526,8 @@ export interface UpgradeVerifyResult {
     | "none"
     | "already-closed"
     | "verified"
+    | "recovered"
+    | "recovery-owed"
     | "aborted"
     | "rollback-requested"
     | "rollback-unavailable";
@@ -453,41 +562,54 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
   if (request === undefined) return { handled: "none" };
   const state = classifyUpgrade(entries);
 
-  const journal = (entry: Omit<UpgradeJournalEntry, "at">): void => {
+  const journal = (entry: Omit<UpgradeJournalEntry, "at">, at = now()): void => {
     const write = deps.journal ?? ((line: UpgradeJournalEntry) => appendJournal(root, line));
-    write({ ...entry, version: entry.version ?? request.version, at: new Date(now()).toISOString() });
+    write({ ...entry, version: entry.version ?? request.version, at: new Date(at).toISOString() });
   };
 
   // ---------------------------------------------------------------- terminal
   if (state.kind === "closed") {
-    // The transaction is already terminal (verified, rolled back, aborted,
-    // rollback requested). One thing may still be owed: the install paused
-    // dispatch, and a crash took the process that should have resumed it. The
-    // resume is safe to redo — it only acts while the sentinel is still the
-    // install's own — so it runs on every visit to a closed request.
-    resumeUpgradePause(deps, request);
+    // A closed failure is still a safety hold: rollback-requested means the
+    // rollback unit has not landed yet, while rollback-unavailable and
+    // rollback-failed mean the host may contain mixed surfaces. Only outcomes
+    // that prove the transaction safe may release its pause.
+    const outcome = newestOutcome(progression);
+    const parsedOutcomeAt = outcome === undefined ? Number.NaN : Date.parse(outcome.at);
+    const outcomeAt = Number.isFinite(parsedOutcomeAt) ? parsedOutcomeAt : undefined;
+    if (RECOVERY_CHECK_OUTCOMES.has(state.phase)) {
+      return verifyRecoveredUpgrade(deps, request, state.phase, outcomeAt, journal, now);
+    }
+    if (RECOVERY_HOLD_OUTCOMES.has(state.phase)) deps.holdGlobal();
+    if (RESUMABLE_OUTCOMES.has(state.phase) && outcomeAt !== undefined) {
+      resumeUpgradePause(deps, request, outcomeAt);
+    }
     return { handled: "already-closed", version: request.version, detail: state.phase };
   }
 
   // ------------------------------------------------------------ never started
   if (state.kind === "no-start") {
-    // The unit died before journaling a single phase. Nothing was installed;
-    // the sentinel can still exist if it crashed between pausing and writing
-    // the pause line — clear it, close the request, and page tier 2 that the
-    // fleet could not install its own fix.
-    resumeUpgradePause(deps, request);
+    // The unit died before journaling a single phase. Nothing was installed,
+    // but no phase recorded the exact pause instance either. Close the request
+    // without guessing that an upgrade-owned sentinel still belongs to it.
+    const abortedAt = now();
     deps.enqueue({
       project: deps.projectName,
       kind: "tier2",
       body: [
         `Fleet upgrade to omp-conductor@${request.version} never started (journal: ${upgradeJournalPath(root)}).`,
-        "The detached install unit died before touching any surface; dispatch was not paused.",
+        "The detached install unit died before touching any surface.",
+        "Dispatch may still hold an unidentifiable install pause; check `status` and clear it explicitly with `omp-conductor resume`.",
         `Re-run the request, or install by hand with \`omp-conductor upgrade --to ${request.version}\`.`,
       ].join("\n"),
-      at: now(),
+      at: abortedAt,
       dedupeKey: `upgrade:${request.version}:aborted`,
     });
-    journal({ kind: "outcome", phase: "aborted", ok: false, detail: "requested upgrade never started a phase" });
+    journal({
+      kind: "outcome",
+      phase: "aborted",
+      ok: false,
+      detail: "requested upgrade never started a phase",
+    }, abortedAt);
     return { handled: "aborted", version: request.version, detail: "no phase was ever journaled" };
   }
 
@@ -518,16 +640,9 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
     );
   }
 
-  // Good. Resume dispatch only when the install paused it, and only while the
-  // sentinel is still the install's own — an operator's later hold is never
-  // lifted by an upgrade closing.
-  if (request.initialPaused === false) {
-    const owned = deps.pause(request.pauseKey);
-    if (owned !== undefined && owned.source === "upgrade") {
-      deps.resume(request.pauseKey);
-      deps.log(`upgrade verified — dispatch resumed (${request.pauseKey ?? "global"})`);
-    }
-  }
+  // Good. Persist the evidence before releasing anything: if this process dies
+  // after the outcome, the next tick can finish the idempotent resume.
+  const verifiedAt = now();
   deps.enqueue({
     project: deps.projectName,
     kind: "material",
@@ -538,23 +653,118 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
       "",
       "Dispatch restored to its prior state.",
     ].join("\n"),
-    at: now(),
+    at: verifiedAt,
     dedupeKey: `upgrade:${request.version}:verified`,
   });
-  journal({ kind: "outcome", phase: "verified", ok: true, checks });
+  journal({ kind: "outcome", phase: "verified", ok: true, checks }, verifiedAt);
+  const paused = deps.pause(request.pauseKey);
+  resumeUpgradePause(deps, request, verifiedAt);
+  if (paused !== undefined && deps.pause(request.pauseKey) === undefined) {
+    deps.log(`upgrade verified — dispatch resumed (${request.pauseKey ?? "global"})`);
+  }
   return { handled: "verified", version: request.version };
 }
 
+async function verifyRecoveredUpgrade(
+  deps: UpgradeVerifyDeps,
+  request: PendingUpgradeRequest,
+  failedPhase: string,
+  failedOutcomeAt: number | undefined,
+  journal: (entry: Omit<UpgradeJournalEntry, "at">, at?: number) => void,
+  now: () => number,
+): Promise<UpgradeVerifyResult> {
+  const globalPause = deps.pause();
+  const globalRecoverySince =
+    globalPause?.source === "upgrade-recovery" ? globalPause.since : undefined;
+  const checks: UpgradeCheck[] = [];
+  const expected = request.previous;
+  if (expected === undefined) {
+    checks.push({ name: "installed-surfaces", ok: false, detail: "pre-upgrade snapshot is missing" });
+  } else {
+    try {
+      const actual = await deps.surfaces();
+      for (const [name, wanted, found] of [
+        ["cli", expected.cliVersion, actual.cliVersion],
+        ["omp", expected.ompVersion, actual.ompVersion],
+        ["herdr", expected.herdrSource, actual.herdrSource],
+      ] as const) {
+        checks.push({
+          name: `surface:${name}`,
+          ok: wanted === found,
+          detail: `expected ${wanted ?? "absent"}; found ${found ?? "absent"}`,
+        });
+      }
+    } catch (err) {
+      checks.push({
+        name: "installed-surfaces",
+        ok: false,
+        detail: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+  const failed = checks.find((check) => !check.ok);
+  if (failed !== undefined) {
+    // A recovery verdict owns the host-wide gate. Re-establish it on every
+    // owed pass so a legacy failed journal or a lost sentinel cannot admit.
+    deps.holdGlobal();
+    return {
+      handled: "recovery-owed",
+      version: request.version,
+      detail: `${failedPhase}; ${failed.name}: ${failed.detail ?? "failed"}`,
+    };
+  }
+
+  const recoveredAt = now();
+  deps.enqueue({
+    project: deps.projectName,
+    kind: "material",
+    body: [
+      `The failed upgrade to omp-conductor@${request.version} is recovered.`,
+      "",
+      ...checks.map((check) => `ok  ${check.name} — ${check.detail ?? "matched"}`),
+      "",
+      "Every installed surface matches the durable pre-upgrade snapshot; dispatch restored to its prior state.",
+    ].join("\n"),
+    at: recoveredAt,
+    dedupeKey: `upgrade:${request.version}:recovered`,
+  });
+  journal({
+    kind: "outcome",
+    phase: "recovered",
+    ok: true,
+    detail: `verified recovery after ${failedPhase}`,
+    checks,
+  }, recoveredAt);
+  resumeUpgradePause(deps, request, failedOutcomeAt);
+  deps.releaseGlobalRecovery(globalRecoverySince);
+  return { handled: "recovered", version: request.version, detail: failedPhase };
+}
+
 /**
  * Lift the install's own pause sentinel, and only that one. `pauseKey` is the
  * sentinel the engine paused; a fixed `initialPaused` true means the fleet was
  * already held and the install never paused it, so nothing is lifted. A pause
  * that an operator re-created (source changed) is left standing.
+ *
+ * New journals carry the exact pause creation instant, so identical
+ * provenance can never make a newer transaction look owned. `notAfter` is the
+ * compatibility bound for older journals that predate that identity field:
+ * only a pause that existed before the safe terminal outcome may be released.
  */
-function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequest): void {
+function resumeUpgradePause(
+  deps: UpgradeVerifyDeps,
+  request: PendingUpgradeRequest,
+  notAfter?: number,
+): void {
   if (request.initialPaused === true) return;
   const owned = deps.pause(request.pauseKey);
-  if (owned !== undefined && owned.source === "upgrade") {
+  const sameTransaction =
+    request.pauseSince !== undefined
+      ? owned !== undefined &&
+        owned.since >= request.pauseSince &&
+        (notAfter === undefined || owned.since <= notAfter)
+      : notAfter !== undefined && owned !== undefined && owned.since <= notAfter;
+  if (owned?.source === "upgrade" && sameTransaction) {
     deps.resume(request.pauseKey);
   }
 }
@@ -687,6 +897,10 @@ async function triggerUpgradeRollback(
   at: number,
   why: string,
 ): Promise<UpgradeVerifyResult> {
+  // Every install surface is host-wide even when the request named one
+  // project. Once recovery is required, the safety fence must gate every
+  // project served by the shared daemon.
+  deps.holdGlobal();
   // The marker is the spawn guard: present-and-ok means a rollback unit is
   // already on its way and a crash after the spawn must not spawn a second
   // one. Present-and-failed means nothing started, so a retry is a first

package/src/upgrade.ts

@@ -129,6 +129,8 @@ export interface UpgradeDeps {
    */
   pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
   setPaused(value: boolean, project?: string): void;
+  /** Host-global recovery fence for host-wide install surfaces. */
+  setGlobalRecoveryPaused(value: boolean): void;
   restartDaemon(): Promise<void>;
   sleep(ms: number): Promise<void>;
   /**
@@ -205,6 +207,14 @@ export const DEFAULT_DEPS: UpgradeDeps = {
   pauseState: (project) => pauseInstance(project),
   setPaused: (v, project) =>
     setPaused(v, { source: "upgrade", reason: "upgrade, draining" }, project),
+  setGlobalRecoveryPaused: (v) => {
+    const current = pauseInstance();
+    if (v && current === undefined) {
+      setPaused(true, { source: "upgrade-recovery", reason: "upgrade recovery verification required" });
+    } else if (!v && current?.source === "upgrade-recovery") {
+      setPaused(false);
+    }
+  },
   restartDaemon: async () => {
     await restartDaemon({});
   },
@@ -1413,7 +1423,14 @@ export async function upgradeConductor(
     deps.log("safety: pausing new issue claims");
     deps.setPaused(true, scope.pauseKey);
   }
-  journal({ kind: "phase", phase: "paused", surface: "dispatch", ok: true, version: release.version });
+  journal({
+    kind: "phase",
+    phase: "paused",
+    surface: "dispatch",
+    ok: true,
+    version: release.version,
+    pauseSince: deps.pauseState(scope.pauseKey)?.since,
+  });
   deps.log("drain: waiting for live omp worker sessions");
   try {
     await waitForDrain(deps, scope);
@@ -1595,6 +1612,7 @@ export async function upgradeConductor(
     } catch {
       deps.setPaused(true, scope.pauseKey);
     }
+    deps.setGlobalRecoveryPaused(true);
     const failure = err instanceof Error ? err.message : String(err);
     journal({
       kind: "phase",
@@ -1779,6 +1797,10 @@ export async function rollbackFromJournal(
     configBefore,
     request.configBackup,
   );
+  // The restored bytes remain host-wide unverified state until the returning
+  // daemon reads all three surfaces. A project-scoped request cannot leave its
+  // siblings admitting in that interval.
+  deps.setGlobalRecoveryPaused(true);
 
   const restoredVersion = previous.cliVersion;
   appendJournal(root, {