omp-conductor 0.19.4 → 0.19.6

package/src/upgrade-verify.ts

@@ -25,7 +25,7 @@ import { readFileSync } from "node:fs";
 import { loadConfig, stateDir } from "./config.ts";
 import { DEFAULT_PORT } from "./lifecycle.ts";
 import { dbPath, openStore } from "./store.ts";
-import { appendJournal, readUpgradeJournal, upgradeJournalPath, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
+import { appendJournal, readUpgradeJournal, upgradeJournalPath, type JournalSurfaces, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
 import type { DoctorReport } from "./doctor.ts";
 import type { FleetLayers } from "./status-render.ts";
 import type { ReportDraft, ReportKind } from "./types.ts";
@@ -135,6 +135,17 @@ function newestRequestIndex(entries: readonly UpgradeJournalEntry[]): number {
   return -1;
 }
 
+/** Newest terminal outcome in one request's progression, without copying it. */
+function newestOutcome(
+  progression: readonly UpgradeJournalEntry[],
+): UpgradeJournalEntry | undefined {
+  for (let i = progression.length - 1; i >= 0; i--) {
+    const entry = progression[i]!;
+    if (entry.kind === "outcome") return entry;
+  }
+  return undefined;
+}
+
 /**
  * Classify one journal: no request, a request whose outcome is already
  * terminal, or a request still in flight — with the phase evidence that says
@@ -152,7 +163,7 @@ export function classifyUpgrade(
   const start = newestRequestIndex(entries);
   if (start === -1) return { kind: "none" };
   const progression = entries.slice(start);
-  const outcome = [...progression].reverse().find((entry) => entry.kind === "outcome");
+  const outcome = newestOutcome(progression);
   if (outcome !== undefined) {
     return { kind: "closed", phase: outcome.phase ?? "outcome" };
   }
@@ -167,15 +178,36 @@ export function classifyUpgrade(
   return { kind: "incomplete" };
 }
 
+/** Terminal outcomes that prove no broken or mixed install remains active. */
+const RESUMABLE_OUTCOMES = new Set([
+  "already-current",
+  "recovered",
+  "recovery-acknowledged",
+  "verified",
+]);
+/** Outcomes that require a durable host-wide safety fence. */
+const RECOVERY_HOLD_OUTCOMES = new Set([
+  "rollback-failed",
+  "rollback-requested",
+  "rollback-unavailable",
+  "rolled-back",
+]);
+/** Closed rollback outcomes whose installed surfaces must be re-verified. */
+const RECOVERY_CHECK_OUTCOMES = new Set(["rollback-failed", "rolled-back"]);
+
 /** One pending request plus the snapshot its verifier needs. */
 export interface PendingUpgradeRequest {
   version: string;
   gitHead?: string;
+  pauseSince?: number;
   initialPaused?: boolean;
   pauseKey?: string;
   selectors: readonly (string | undefined)[];
+  installHolder?: UpgradeJournalEntry["installHolder"];
+  installProjects?: readonly string[];
   initial?: UpgradeJournalEntry["initial"];
   configBackup?: string;
+  previous?: JournalSurfaces;
   /**
    * Epoch-ms when the install began (the journal snapshot's write time): the
    * deadline the live orchestrator session must have restarted after (#832).
@@ -202,19 +234,104 @@ export function pendingUpgradeRequest(
   const request = progression[0]!;
   if (request.ok === false) return undefined;
   const snapshot = progression.find((entry) => entry.kind === "snapshot");
+  const pausePhase = progression.find(
+    (entry) => entry.kind === "phase" && entry.phase === "paused",
+  );
   const snapshotAt = snapshot === undefined ? undefined : Date.parse(snapshot.at);
   return {
     version: request.version ?? "unknown",
     gitHead: request.gitHead,
     initialPaused: snapshot?.initialPaused,
     pauseKey: snapshot?.pauseKey,
-    selectors: (snapshot?.selectors ?? []).map((selector) => selector ?? undefined),
+    selectors: (snapshot?.selectors ?? request.selectors ?? []).map((selector) => selector ?? undefined),
+    pauseSince: pausePhase?.pauseSince,
     initial: snapshot?.initial,
     configBackup: snapshot?.configBackup,
+    previous: snapshot?.previous,
+    installHolder: request.installHolder,
+    installProjects: request.installProjects,
     ...(snapshotAt !== undefined && Number.isFinite(snapshotAt) ? { reloadAfterMs: snapshotAt } : {}),
   };
 }
 
+function reportProjects(request: PendingUpgradeRequest): readonly (string | undefined)[] {
+  return request.installProjects ?? request.selectors;
+}
+
+function installAttribution(request: PendingUpgradeRequest): string {
+  const projects =
+    reportProjects(request).length === 0
+      ? "all configured projects"
+      : reportProjects(request).map((selector) => selector ?? "(default)").join(", ");
+  return `Host-global install holder: ${request.installHolder ?? "unknown"}; affected projects: ${projects}.`;
+}
+
+export interface UpgradeRecoveryStatus {
+  failedVersion: string;
+  phase: string;
+  journalPath: string;
+  verificationOwed: string;
+}
+
+/**
+ * The unresolved safety hold represented by the newest journal request.
+ * Status reads this same lifecycle verdict as the verifier; it never guesses
+ * from the pause reason alone.
+ */
+export function upgradeRecoveryStatus(
+  entries: readonly UpgradeJournalEntry[],
+  root = stateDir(),
+): UpgradeRecoveryStatus | undefined {
+  const request = pendingUpgradeRequest(entries);
+  const state = classifyUpgrade(entries);
+  if (
+    request === undefined ||
+    state.kind !== "closed" ||
+    !RECOVERY_HOLD_OUTCOMES.has(state.phase)
+  ) {
+    return undefined;
+  }
+  const expected = request.previous;
+  const surfaces =
+    expected === undefined
+      ? "the pre-upgrade surface snapshot is missing"
+      : [
+          `CLI=${expected.cliVersion}`,
+          `omp=${expected.ompVersion ?? "absent"}`,
+          `herdr=${expected.herdrSource ?? "absent"}`,
+        ].join(", ");
+  const verificationOwed =
+    state.phase === "rollback-requested"
+      ? `detached rollback completion, then installed-surface verification against ${surfaces}`
+      : `installed-surface verification against ${surfaces}`;
+  return {
+    failedVersion: request.version,
+    phase: state.phase,
+    journalPath: upgradeJournalPath(root),
+    verificationOwed,
+  };
+}
+
+/**
+ * Record the operator's explicit choice to release an unresolved upgrade
+ * recovery fence. `resume` calls this before removing an upgrade-owned pause,
+ * so the append-only journal distinguishes acknowledgement from verification.
+ */
+export function acknowledgeUpgradeRecovery(root = stateDir()): boolean {
+  const entries = readUpgradeJournal(root);
+  const recovery = upgradeRecoveryStatus(entries, root);
+  if (recovery === undefined) return false;
+  appendJournal(root, {
+    at: new Date().toISOString(),
+    kind: "outcome",
+    phase: "recovery-acknowledged",
+    ok: true,
+    version: recovery.failedVersion,
+    detail: `operator resumed dispatch with ${recovery.verificationOwed} still owed`,
+  });
+  return true;
+}
+
 // ---------------------------------------------------------------------------
 // The live-orchestrator session attestation (#832)
 // ---------------------------------------------------------------------------
@@ -387,12 +504,18 @@ export interface UpgradeVerifyDeps {
   layers(project?: string): FleetLayers;
   health(port: number): Promise<{ ok: boolean; body?: string }>;
   doctor(projectName: string): Promise<DoctorReport>;
+  /** Read all three package/plugin identities from the live host. */
+  surfaces(): Promise<JournalSurfaces>;
   /** The durable outbox enqueue — one row, daemon-owned delivery. */
   enqueue(draft: ReportDraft): void;
   /** Read one pause sentinel (who set it, when) — daemon-owned state. */
   pause(project?: string): { source: string; reason?: string; since: number } | undefined;
   /** Lift the pause sentinel for a project, clearing the legacy global too. */
   resume(project?: string): void;
+  /** Escalate a project pause to the host-global safety fence. */
+  holdGlobal(): void;
+  /** Remove only the exact dedicated recovery fence observed by this pass. */
+  releaseGlobalRecovery(expectedSince: number | undefined): void;
   /** Start the detached rollback unit for a version. */
   launchRollback(
     version: string,
@@ -419,6 +542,8 @@ export interface UpgradeVerifyResult {
     | "none"
     | "already-closed"
     | "verified"
+    | "recovered"
+    | "recovery-owed"
     | "aborted"
     | "rollback-requested"
     | "rollback-unavailable";
@@ -453,41 +578,55 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
   if (request === undefined) return { handled: "none" };
   const state = classifyUpgrade(entries);
 
-  const journal = (entry: Omit<UpgradeJournalEntry, "at">): void => {
+  const journal = (entry: Omit<UpgradeJournalEntry, "at">, at = now()): void => {
     const write = deps.journal ?? ((line: UpgradeJournalEntry) => appendJournal(root, line));
-    write({ ...entry, version: entry.version ?? request.version, at: new Date(now()).toISOString() });
+    write({ ...entry, version: entry.version ?? request.version, at: new Date(at).toISOString() });
   };
 
   // ---------------------------------------------------------------- terminal
   if (state.kind === "closed") {
-    // The transaction is already terminal (verified, rolled back, aborted,
-    // rollback requested). One thing may still be owed: the install paused
-    // dispatch, and a crash took the process that should have resumed it. The
-    // resume is safe to redo — it only acts while the sentinel is still the
-    // install's own — so it runs on every visit to a closed request.
-    resumeUpgradePause(deps, request);
+    // A closed failure is still a safety hold: rollback-requested means the
+    // rollback unit has not landed yet, while rollback-unavailable and
+    // rollback-failed mean the host may contain mixed surfaces. Only outcomes
+    // that prove the transaction safe may release its pause.
+    const outcome = newestOutcome(progression);
+    const parsedOutcomeAt = outcome === undefined ? Number.NaN : Date.parse(outcome.at);
+    const outcomeAt = Number.isFinite(parsedOutcomeAt) ? parsedOutcomeAt : undefined;
+    if (RECOVERY_CHECK_OUTCOMES.has(state.phase)) {
+      return verifyRecoveredUpgrade(deps, request, state.phase, outcomeAt, journal, now);
+    }
+    if (RECOVERY_HOLD_OUTCOMES.has(state.phase)) deps.holdGlobal();
+    if (RESUMABLE_OUTCOMES.has(state.phase) && outcomeAt !== undefined) {
+      resumeUpgradePause(deps, request, outcomeAt);
+    }
     return { handled: "already-closed", version: request.version, detail: state.phase };
   }
 
   // ------------------------------------------------------------ never started
   if (state.kind === "no-start") {
-    // The unit died before journaling a single phase. Nothing was installed;
-    // the sentinel can still exist if it crashed between pausing and writing
-    // the pause line — clear it, close the request, and page tier 2 that the
-    // fleet could not install its own fix.
-    resumeUpgradePause(deps, request);
+    // The unit died before journaling a single phase. Nothing was installed,
+    // but no phase recorded the exact pause instance either. Close the request
+    // without guessing that an upgrade-owned sentinel still belongs to it.
+    const abortedAt = now();
     deps.enqueue({
-      project: deps.projectName,
+      project: reportProject(reportProjects(request)),
       kind: "tier2",
       body: [
         `Fleet upgrade to omp-conductor@${request.version} never started (journal: ${upgradeJournalPath(root)}).`,
-        "The detached install unit died before touching any surface; dispatch was not paused.",
+        installAttribution(request),
+        "The detached install unit died before touching any surface.",
+        "Dispatch may still hold an unidentifiable install pause; check `status` and clear it explicitly with `omp-conductor resume`.",
         `Re-run the request, or install by hand with \`omp-conductor upgrade --to ${request.version}\`.`,
       ].join("\n"),
-      at: now(),
+      at: abortedAt,
       dedupeKey: `upgrade:${request.version}:aborted`,
     });
-    journal({ kind: "outcome", phase: "aborted", ok: false, detail: "requested upgrade never started a phase" });
+    journal({
+      kind: "outcome",
+      phase: "aborted",
+      ok: false,
+      detail: "requested upgrade never started a phase",
+    }, abortedAt);
     return { handled: "aborted", version: request.version, detail: "no phase was ever journaled" };
   }
 
@@ -518,43 +657,133 @@ export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<Upg
     );
   }
 
-  // Good. Resume dispatch only when the install paused it, and only while the
-  // sentinel is still the install's own — an operator's later hold is never
-  // lifted by an upgrade closing.
-  if (request.initialPaused === false) {
-    const owned = deps.pause(request.pauseKey);
-    if (owned !== undefined && owned.source === "upgrade") {
-      deps.resume(request.pauseKey);
-      deps.log(`upgrade verified — dispatch resumed (${request.pauseKey ?? "global"})`);
-    }
-  }
+  // Good. Persist the evidence before releasing anything: if this process dies
+  // after the outcome, the next tick can finish the idempotent resume.
+  const verifiedAt = now();
   deps.enqueue({
-    project: deps.projectName,
+    project: reportProject(reportProjects(request)),
     kind: "material",
     body: [
       `omp-conductor@${request.version} is installed and verified on the first tick after the restart.`,
+      installAttribution(request),
       "",
       ...checks.map((check) => `${check.ok ? "ok" : "FAIL"}  ${check.name}${check.detail === undefined ? "" : ` — ${check.detail}`}`),
       "",
       "Dispatch restored to its prior state.",
     ].join("\n"),
-    at: now(),
+    at: verifiedAt,
     dedupeKey: `upgrade:${request.version}:verified`,
   });
-  journal({ kind: "outcome", phase: "verified", ok: true, checks });
+  journal({ kind: "outcome", phase: "verified", ok: true, checks }, verifiedAt);
+  const paused = deps.pause(request.pauseKey);
+  resumeUpgradePause(deps, request, verifiedAt);
+  if (paused !== undefined && deps.pause(request.pauseKey) === undefined) {
+    deps.log(`upgrade verified — dispatch resumed (${request.pauseKey ?? "global"})`);
+  }
   return { handled: "verified", version: request.version };
 }
 
+async function verifyRecoveredUpgrade(
+  deps: UpgradeVerifyDeps,
+  request: PendingUpgradeRequest,
+  failedPhase: string,
+  failedOutcomeAt: number | undefined,
+  journal: (entry: Omit<UpgradeJournalEntry, "at">, at?: number) => void,
+  now: () => number,
+): Promise<UpgradeVerifyResult> {
+  const globalPause = deps.pause();
+  const globalRecoverySince =
+    globalPause?.source === "upgrade-recovery" ? globalPause.since : undefined;
+  const checks: UpgradeCheck[] = [];
+  const expected = request.previous;
+  if (expected === undefined) {
+    checks.push({ name: "installed-surfaces", ok: false, detail: "pre-upgrade snapshot is missing" });
+  } else {
+    try {
+      const actual = await deps.surfaces();
+      for (const [name, wanted, found] of [
+        ["cli", expected.cliVersion, actual.cliVersion],
+        ["omp", expected.ompVersion, actual.ompVersion],
+        ["herdr", expected.herdrSource, actual.herdrSource],
+      ] as const) {
+        checks.push({
+          name: `surface:${name}`,
+          ok: wanted === found,
+          detail: `expected ${wanted ?? "absent"}; found ${found ?? "absent"}`,
+        });
+      }
+    } catch (err) {
+      checks.push({
+        name: "installed-surfaces",
+        ok: false,
+        detail: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+  const failed = checks.find((check) => !check.ok);
+  if (failed !== undefined) {
+    // A recovery verdict owns the host-wide gate. Re-establish it on every
+    // owed pass so a legacy failed journal or a lost sentinel cannot admit.
+    deps.holdGlobal();
+    return {
+      handled: "recovery-owed",
+      version: request.version,
+      detail: `${failedPhase}; ${failed.name}: ${failed.detail ?? "failed"}`,
+    };
+  }
+
+  const recoveredAt = now();
+  deps.enqueue({
+    project: reportProject(reportProjects(request)),
+    kind: "material",
+    body: [
+      `The failed upgrade to omp-conductor@${request.version} is recovered.`,
+      installAttribution(request),
+      "",
+      ...checks.map((check) => `ok  ${check.name} — ${check.detail ?? "matched"}`),
+      "",
+      "Every installed surface matches the durable pre-upgrade snapshot; dispatch restored to its prior state.",
+    ].join("\n"),
+    at: recoveredAt,
+    dedupeKey: `upgrade:${request.version}:recovered`,
+  });
+  journal({
+    kind: "outcome",
+    phase: "recovered",
+    ok: true,
+    detail: `verified recovery after ${failedPhase}`,
+    checks,
+  }, recoveredAt);
+  resumeUpgradePause(deps, request, failedOutcomeAt);
+  deps.releaseGlobalRecovery(globalRecoverySince);
+  return { handled: "recovered", version: request.version, detail: failedPhase };
+}
+
 /**
  * Lift the install's own pause sentinel, and only that one. `pauseKey` is the
  * sentinel the engine paused; a fixed `initialPaused` true means the fleet was
  * already held and the install never paused it, so nothing is lifted. A pause
  * that an operator re-created (source changed) is left standing.
+ *
+ * New journals carry the exact pause creation instant, so identical
+ * provenance can never make a newer transaction look owned. `notAfter` is the
+ * compatibility bound for older journals that predate that identity field:
+ * only a pause that existed before the safe terminal outcome may be released.
  */
-function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequest): void {
+function resumeUpgradePause(
+  deps: UpgradeVerifyDeps,
+  request: PendingUpgradeRequest,
+  notAfter?: number,
+): void {
   if (request.initialPaused === true) return;
   const owned = deps.pause(request.pauseKey);
-  if (owned !== undefined && owned.source === "upgrade") {
+  const sameTransaction =
+    request.pauseSince !== undefined
+      ? owned !== undefined &&
+        owned.since >= request.pauseSince &&
+        (notAfter === undefined || owned.since <= notAfter)
+      : notAfter !== undefined && owned !== undefined && owned.since <= notAfter;
+  if (owned?.source === "upgrade" && sameTransaction) {
     deps.resume(request.pauseKey);
   }
 }
@@ -687,6 +916,10 @@ async function triggerUpgradeRollback(
   at: number,
   why: string,
 ): Promise<UpgradeVerifyResult> {
+  // Every install surface is host-wide even when the request named one
+  // project. Once recovery is required, the safety fence must gate every
+  // project served by the shared daemon.
+  deps.holdGlobal();
   // The marker is the spawn guard: present-and-ok means a rollback unit is
   // already on its way and a crash after the spawn must not spawn a second
   // one. Present-and-failed means nothing started, so a retry is a first
@@ -711,10 +944,11 @@ async function triggerUpgradeRollback(
     // and must not be quietly mixed.
     journal({ kind: "outcome", phase: "rollback-unavailable", ok: false, detail: spawned.stderr });
     deps.enqueue({
-      project: deps.projectName,
+      project: reportProject(reportProjects(request)),
       kind: "tier2",
       body: [
         `Fleet upgrade to omp-conductor@${request.version} is INCOMPLETE (${why}) and its detached rollback could not start.`,
+        installAttribution(request),
         `Reason: ${spawned.stderr}`,
         "",
         "Surfaces may be at mixed versions; do not resume dispatch.",
@@ -727,10 +961,11 @@ async function triggerUpgradeRollback(
   }
   journal({ kind: "outcome", phase: "rollback-requested", ok: false, detail: why });
   deps.enqueue({
-    project: deps.projectName,
+    project: reportProject(reportProjects(request)),
     kind: "tier2",
     body: [
       `Fleet upgrade to omp-conductor@${request.version} did not verify (${why}) and is being rolled back by ${spawned.unit}.`,
+      installAttribution(request),
       "Dispatch stays paused until the rollback lands and a later tick verifies the old version.",
       "",
       `Journal: ${upgradeJournalPath(root)}`,