omp-conductor 0.15.9 → 0.15.11

package/src/upgrade-verify.ts

@@ -0,0 +1,506 @@
+/**
+ * The durable seam between the detached fleet installer and the daemon that
+ * verifies it (#486).
+ *
+ * `omp-conductor upgrade` has always been a transaction *inside* one process:
+ * it paused, drained, installed, reloaded and verified, and the process that
+ * verified was the process that restarted the fleet. The fleet-installs-itself
+ * path replaces that wrapper, not the transaction: a detached transient unit
+ * performs the install while `upgrade-journal.ts` records every surface it
+ * touches, and the first tick of the post-restart daemon verifies against the
+ * journal — a process that did not run the install, which is the only witness
+ * worth trusting after the restarts.
+ *
+ * This module is a leaf on purpose. The daemon's privileged tick and the
+ * upgrade engine both need the verifier, the request classification and the
+ * transient-unit naming, and they already import each other — `upgrade.ts`
+ * takes its pause helpers from `daemon.ts`, and the daemon runs the engine's
+ * rollback. Putting the shared seams here instead of importing `upgrade.ts`
+ * from the daemon keeps that graph acyclic; everything the verifier touches
+ * that belongs to a host state is injected through `UpgradeVerifyDeps`, so the
+ * only runtime imports are leaves (journal, config, store).
+ */
+
+import { loadConfig, stateDir } from "./config.ts";
+import { DEFAULT_PORT } from "./lifecycle.ts";
+import { dbPath, openStore } from "./store.ts";
+import { appendJournal, readUpgradeJournal, upgradeJournalPath, type UpgradeCheck, type UpgradeJournalEntry } from "./upgrade-journal.ts";
+import type { DoctorReport } from "./doctor.ts";
+import type { FleetLayers } from "./fleet.ts";
+import type { ReportDraft, ReportKind } from "./types.ts";
+
+/** One command run to completion with captured output. The transient unit, the
+ *  post-restart verifier and the rollback unit all run processes this way —
+ *  one named construction site, so the seam tests can inject it everywhere. */
+export interface UpgradeCommandResult {
+  code: number;
+  stdout: string;
+  stderr: string;
+}
+
+export async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
+  try {
+    const child = Bun.spawn([command, ...args], {
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+      env: process.env,
+    });
+    const stdout = new Response(child.stdout).text();
+    const stderr = new Response(child.stderr).text();
+    const code = await child.exited;
+    return { code, stdout: await stdout, stderr: await stderr };
+  } catch (err) {
+    return { code: 127, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+/**
+ * The project a host-wide outcome report is filed under: the first in-scope
+ * project, falling back to the configured first project, then the constant.
+ */
+export function reportProject(selectors: readonly (string | undefined)[]): string {
+  for (const selector of selectors) {
+    if (selector !== undefined) return selector;
+  }
+  try {
+    const projects = loadConfig().projects;
+    if (projects.length > 0) return projects[0]!.name;
+  } catch {
+    // no config: the fallback below stands
+  }
+  return "conductor";
+}
+
+/**
+ * Hand a durable outbox row over in the detached context, where no tick is
+ * alive to do it. The daemon's report outbox owns delivery from here (bounded
+ * retries via the same store rows `omp-conductor report` writes), so a report
+ * survives the process that enqueued it. Refusal to queue is logged and lost
+ * — the journal on disk is the harder of the two records and is always kept.
+ */
+export function enqueueUpgradeReport(
+  kind: ReportKind,
+  project: string,
+  summary: string,
+  lines: string[],
+  dedupeKey: string,
+): void {
+  try {
+    const store = openStore(dbPath());
+    try {
+      store.enqueueReport({
+        project,
+        kind,
+        body: [summary, "", ...lines].join("\n"),
+        at: Date.now(),
+        dedupeKey,
+      });
+    } finally {
+      store.close();
+    }
+  } catch (err) {
+    // logged by nothing: the journal carries the same outcome, and `status`
+    // surfaces the reports block. A full disk must not loop here.
+    process.stderr.write(`upgrade report could not be queued: ${err instanceof Error ? err.message : String(err)}\n`);
+  }
+}
+
+// ---------------------------------------------------------------------------
+// Classifying the journal
+// ---------------------------------------------------------------------------
+
+/** The state a pending request has reached, read off the journal. */
+export type PendingUpgradeState =
+  | { kind: "none" }
+  | { kind: "closed"; phase: string }
+  | { kind: "no-start" }
+  | { kind: "awaiting-verification" }
+  | { kind: "incomplete" }
+  | { kind: "failed" };
+
+/** Index of the newest request line, or -1 when none exists. */
+function newestRequestIndex(entries: readonly UpgradeJournalEntry[]): number {
+  for (let i = entries.length - 1; i >= 0; i--) {
+    if (entries[i]!.kind === "request") return i;
+  }
+  return -1;
+}
+
+/**
+ * Classify one journal: no request, a request whose outcome is already
+ * terminal, or a request still in flight — with the phase evidence that says
+ * how far it got. An outcome is terminal: the verifier writes it last, after
+ * the durable report is already queued, so a later tick reads it and does
+ * nothing rather than re-reporting.
+ *
+ * The journal is append-only across upgrades, so the newest request is the one
+ * under test: an older request's terminal outcome must not close or shadow a
+ * newer request, and only the newest request's progression decides its state.
+ */
+export function classifyUpgrade(
+  entries: readonly UpgradeJournalEntry[],
+): PendingUpgradeState {
+  const start = newestRequestIndex(entries);
+  if (start === -1) return { kind: "none" };
+  const progression = entries.slice(start);
+  const outcome = [...progression].reverse().find((entry) => entry.kind === "outcome");
+  if (outcome !== undefined) {
+    return { kind: "closed", phase: outcome.phase ?? "outcome" };
+  }
+  const phases = progression.filter((entry) => entry.kind === "phase");
+  if (phases.length === 0) return { kind: "no-start" };
+  if (phases.some((entry) => entry.phase === "awaiting-verification" && entry.ok)) {
+    return { kind: "awaiting-verification" };
+  }
+  if (phases.some((entry) => entry.ok === false)) {
+    return { kind: "failed" };
+  }
+  return { kind: "incomplete" };
+}
+
+/** One pending request plus the snapshot its verifier needs. */
+export interface PendingUpgradeRequest {
+  version: string;
+  gitHead?: string;
+  initialPaused?: boolean;
+  pauseKey?: string;
+  selectors: readonly (string | undefined)[];
+  initial?: UpgradeJournalEntry["initial"];
+  configBackup?: string;
+}
+
+/** The newest request plus everything journaled after it: the view one
+ *  transaction owns. Older closed upgrades stay in the journal but do not
+ *  contribute identity or phases to the one in flight. */
+export function requestProgression(
+  entries: readonly UpgradeJournalEntry[],
+): readonly UpgradeJournalEntry[] | undefined {
+  const start = newestRequestIndex(entries);
+  if (start === -1) return undefined;
+  return entries.slice(start);
+}
+
+export function pendingUpgradeRequest(
+  entries: readonly UpgradeJournalEntry[],
+): PendingUpgradeRequest | undefined {
+  const progression = requestProgression(entries);
+  if (progression === undefined) return undefined;
+  const request = progression[0]!;
+  if (request.ok === false) return undefined;
+  const snapshot = progression.find((entry) => entry.kind === "snapshot");
+  return {
+    version: request.version ?? "unknown",
+    gitHead: request.gitHead,
+    initialPaused: snapshot?.initialPaused,
+    pauseKey: snapshot?.pauseKey,
+    selectors: (snapshot?.selectors ?? []).map((selector) => selector ?? undefined),
+    initial: snapshot?.initial,
+    configBackup: snapshot?.configBackup,
+  };
+}
+
+// ---------------------------------------------------------------------------
+// The post-restart verifier
+// ---------------------------------------------------------------------------
+
+/**
+ * Every injectable process/state seam the post-restart verifier touches.
+ *
+ * The verifier runs in the daemon, so everything that belongs to the daemon's
+ * live state (the pause sentinel, the layers, the outbox, the health probe)
+ * is injected rather than imported — the daemon wires its own functions, and
+ * a test drives the whole pass with fakes.
+ */
+export interface UpgradeVerifyDeps {
+  projectName: string;
+  /** The journal directory; defaults to the conductor state dir. */
+  journalRoot?: string;
+  /** Sink for journal lines; defaults to the state-dir journal. */
+  journal?: (entry: UpgradeJournalEntry) => void;
+  run: (command: string, args: readonly string[]) => Promise<UpgradeCommandResult>;
+  layers(project?: string): FleetLayers;
+  health(port: number): Promise<{ ok: boolean; body?: string }>;
+  doctor(projectName: string): Promise<DoctorReport>;
+  /** The durable outbox enqueue — one row, daemon-owned delivery. */
+  enqueue(draft: ReportDraft): void;
+  /** Read one pause sentinel (who set it, when) — daemon-owned state. */
+  pause(project?: string): { source: string; reason?: string; since: number } | undefined;
+  /** Lift the pause sentinel for a project, clearing the legacy global too. */
+  resume(project?: string): void;
+  /** Start the detached rollback unit for a version. */
+  launchRollback(
+    version: string,
+  ): Promise<{ ok: true; unit: string } | { ok: false; stderr: string }>;
+  log(message: string): void;
+  now?(): number;
+}
+
+/** What one verify pass did with the pending request. */
+export interface UpgradeVerifyResult {
+  handled:
+    | "none"
+    | "already-closed"
+    | "verified"
+    | "aborted"
+    | "rollback-requested"
+    | "rollback-unavailable";
+  version?: string;
+  detail?: string;
+}
+
+/**
+ * The first-tick verification for a fleet-initiated install (#486).
+ *
+ * The installing process journals progress and stops; whichever process comes
+ * back — the restarted daemon's first tick — reads that journal and confirms
+ * independently, then closes the transaction on the evidence: the installed
+ * version matches the request, `/healthz` answers, ticks and pane match the
+ * layers the install began under, and `doctor` has no failing finding.
+ *
+ * Every action is ordered for a crash: the durable outbox row goes in before
+ * the terminal journal outcome, and the outcome's dedupe key makes the
+ * at-least-once re-enqueue a no-op — so a tick that died mid-close re-runs
+ * the whole pass on the next tick without doubling the report or the rollback
+ * unit. The rollback unit is spawned at most once: a `rollback` phase marker
+ * journaled with its outcome is the crash guard, and only a marker that
+ * records a failed spawn (nothing was started) may retry.
+ */
+export async function verifyPendingUpgrade(deps: UpgradeVerifyDeps): Promise<UpgradeVerifyResult> {
+  const root = deps.journalRoot ?? stateDir();
+  const now = deps.now ?? Date.now;
+  const entries = readUpgradeJournal(root);
+  const progression = requestProgression(entries);
+  if (progression === undefined) return { handled: "none" };
+  const request = pendingUpgradeRequest(entries);
+  if (request === undefined) return { handled: "none" };
+  const state = classifyUpgrade(entries);
+
+  const journal = (entry: Omit<UpgradeJournalEntry, "at">): void => {
+    const write = deps.journal ?? ((line: UpgradeJournalEntry) => appendJournal(root, line));
+    write({ ...entry, version: entry.version ?? request.version, at: new Date(now()).toISOString() });
+  };
+
+  // ---------------------------------------------------------------- terminal
+  if (state.kind === "closed") {
+    // The transaction is already terminal (verified, rolled back, aborted,
+    // rollback requested). One thing may still be owed: the install paused
+    // dispatch, and a crash took the process that should have resumed it. The
+    // resume is safe to redo — it only acts while the sentinel is still the
+    // install's own — so it runs on every visit to a closed request.
+    resumeUpgradePause(deps, request);
+    return { handled: "already-closed", version: request.version, detail: state.phase };
+  }
+
+  // ------------------------------------------------------------ never started
+  if (state.kind === "no-start") {
+    // The unit died before journaling a single phase. Nothing was installed;
+    // the sentinel can still exist if it crashed between pausing and writing
+    // the pause line — clear it, close the request, and page tier 2 that the
+    // fleet could not install its own fix.
+    resumeUpgradePause(deps, request);
+    deps.enqueue({
+      project: deps.projectName,
+      kind: "tier2",
+      body: [
+        `Fleet upgrade to omp-conductor@${request.version} never started (journal: ${upgradeJournalPath(root)}).`,
+        "The detached install unit died before touching any surface; dispatch was not paused.",
+        `Re-run the request, or install by hand with \`omp-conductor upgrade --to ${request.version}\`.`,
+      ].join("\n"),
+      at: now(),
+      dedupeKey: `upgrade:${request.version}:aborted`,
+    });
+    journal({ kind: "outcome", phase: "aborted", ok: false, detail: "requested upgrade never started a phase" });
+    return { handled: "aborted", version: request.version, detail: "no phase was ever journaled" };
+  }
+
+  // ----------------------------------------------- incomplete or failed mid-run
+  if (state.kind === "incomplete" || state.kind === "failed") {
+    // The installer died mid-transaction or journaled a failure it could not
+    // roll back itself. Surfaces may be mixed; the fixed answer is the
+    // snapshot rollback, detached, plus a tier-2 page naming the journal.
+    deps.log(
+      `upgrade to ${request.version} left an ${state.kind} journal — triggering the detached rollback`,
+    );
+    return triggerUpgradeRollback(deps, root, journal, request, progression, now(), "mid-flight failure");
+  }
+
+  // ------------------------------------------------------- awaiting verification
+  const checks: UpgradeCheck[] = [];
+  await runUpgradeChecks(deps, request, checks);
+  const failed = checks.find((check) => !check.ok);
+  if (failed !== undefined) {
+    return triggerUpgradeRollback(
+      deps,
+      root,
+      journal,
+      request,
+      progression,
+      now(),
+      `verification failed on ${failed.name} (${failed.detail ?? "no detail"})`,
+    );
+  }
+
+  // Good. Resume dispatch only when the install paused it, and only while the
+  // sentinel is still the install's own — an operator's later hold is never
+  // lifted by an upgrade closing.
+  if (request.initialPaused === false) {
+    const owned = deps.pause(request.pauseKey);
+    if (owned !== undefined && owned.source === "upgrade") {
+      deps.resume(request.pauseKey);
+      deps.log(`upgrade verified — dispatch resumed (${request.pauseKey ?? "global"})`);
+    }
+  }
+  deps.enqueue({
+    project: deps.projectName,
+    kind: "material",
+    body: [
+      `omp-conductor@${request.version} is installed and verified on the first tick after the restart.`,
+      "",
+      ...checks.map((check) => `${check.ok ? "ok" : "FAIL"}  ${check.name}${check.detail === undefined ? "" : ` — ${check.detail}`}`),
+      "",
+      "Dispatch restored to its prior state.",
+    ].join("\n"),
+    at: now(),
+    dedupeKey: `upgrade:${request.version}:verified`,
+  });
+  journal({ kind: "outcome", phase: "verified", ok: true, checks });
+  return { handled: "verified", version: request.version };
+}
+
+/**
+ * Lift the install's own pause sentinel, and only that one. `pauseKey` is the
+ * sentinel the engine paused; a fixed `initialPaused` true means the fleet was
+ * already held and the install never paused it, so nothing is lifted. A pause
+ * that an operator re-created (source changed) is left standing.
+ */
+function resumeUpgradePause(deps: UpgradeVerifyDeps, request: PendingUpgradeRequest): void {
+  if (request.initialPaused === true) return;
+  const owned = deps.pause(request.pauseKey);
+  if (owned !== undefined && owned.source === "upgrade") {
+    deps.resume(request.pauseKey);
+  }
+}
+
+/**
+ * The independent checks the returning process runs. Each compares a live
+ * fact with the journal's record of the world the install began in:
+ *
+ *  - `version` — the installed CLI (all three surfaces are pinned to one
+ *    release by the engine, so the CLI read stands for the set);
+ *  - `ticks` / `pane` / `herdr` — delta against the pre-install layers, the
+ *    same predicate the in-process upgrade used, so a fleet that was armed
+ *    stays armed and a live pane stays live;
+ *  - `health` — the daemon that came back answers `/healthz`;
+ *  - `doctor` — no failing finding.
+ */
+export async function runUpgradeChecks(
+  deps: UpgradeVerifyDeps,
+  request: PendingUpgradeRequest,
+  checks: UpgradeCheck[],
+): Promise<void> {
+  const version = await deps.run("omp-conductor", ["--version"]);
+  checks.push({
+    name: "version",
+    ok: version.code === 0 && version.stdout.trim() === request.version,
+    detail: version.code === 0 ? version.stdout.trim() : version.stderr.trim() || `exit ${version.code}`,
+  });
+
+  const layers = deps.layers(deps.projectName);
+  const before = request.initial;
+  if (before !== undefined) {
+    checks.push({ name: "ticks", ok: layers.ticks === before.ticks, detail: `${before.ticks} → ${layers.ticks}` });
+    checks.push({ name: "pane", ok: layers.pane === before.pane, detail: `${before.pane} → ${layers.pane}` });
+    if (before.herdr === "active") {
+      checks.push({ name: "herdr", ok: layers.herdr === "active", detail: layers.herdr });
+    }
+  }
+  checks.push({
+    name: "health",
+    ok: layers.daemon.running && (await liveHealth(deps, layers.daemon.port ?? DEFAULT_PORT)).ok,
+    detail: layers.daemon.running ? undefined : "no daemon record",
+  });
+
+  const doctor = await deps.doctor(deps.projectName);
+  const failing = doctor.findings.filter((finding) => finding.status === "fail");
+  checks.push({
+    name: "doctor",
+    ok: failing.length === 0,
+    detail: failing.length === 0 ? `status ${doctor.status}` : `${failing[0]!.id}: ${failing[0]!.summary}`,
+  });
+}
+
+async function liveHealth(
+  deps: UpgradeVerifyDeps,
+  port: number,
+): Promise<{ ok: boolean; body?: string }> {
+  try {
+    return await deps.health(port);
+  } catch (err) {
+    return { ok: false, body: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+/** Spawn the detached rollback once and page the evidence, whatever the cause. */
+async function triggerUpgradeRollback(
+  deps: UpgradeVerifyDeps,
+  root: string,
+  journal: (entry: Omit<UpgradeJournalEntry, "at">) => void,
+  request: PendingUpgradeRequest,
+  progress: readonly UpgradeJournalEntry[],
+  at: number,
+  why: string,
+): Promise<UpgradeVerifyResult> {
+  // The marker is the spawn guard: present-and-ok means a rollback unit is
+  // already on its way and a crash after the spawn must not spawn a second
+  // one. Present-and-failed means nothing started, so a retry is a first
+  // attempt, not a double. Scoped to this request's progression — an older
+  // upgrade's rollback marker must not suppress a new one.
+  const marker = progress.find((entry) => entry.kind === "phase" && entry.phase === "rollback");
+  let spawned: { ok: true; unit: string } | { ok: false; stderr: string };
+  if (marker?.ok === true) {
+    spawned = { ok: true, unit: marker.unit ?? "omp-conductor-upgrade-rollback" };
+  } else {
+    spawned = await deps.launchRollback(request.version);
+    journal({
+      kind: "phase",
+      phase: "rollback",
+      ok: spawned.ok,
+      ...(spawned.ok ? { unit: spawned.unit } : { detail: spawned.stderr }),
+    });
+  }
+  if (!spawned.ok) {
+    // Nothing was spawned, so the next tick may legitimately retry a first
+    // attempt. Page with the journal path — the fleet may be half-upgraded
+    // and must not be quietly mixed.
+    journal({ kind: "outcome", phase: "rollback-unavailable", ok: false, detail: spawned.stderr });
+    deps.enqueue({
+      project: deps.projectName,
+      kind: "tier2",
+      body: [
+        `Fleet upgrade to omp-conductor@${request.version} is INCOMPLETE (${why}) and its detached rollback could not start.`,
+        `Reason: ${spawned.stderr}`,
+        "",
+        "Surfaces may be at mixed versions; do not resume dispatch.",
+        `Restore manually from ${upgradeJournalPath(root)}.`,
+      ].join("\n"),
+      at,
+      dedupeKey: `upgrade:${request.version}:rollback-unavailable`,
+    });
+    return { handled: "rollback-unavailable", version: request.version, detail: spawned.stderr };
+  }
+  journal({ kind: "outcome", phase: "rollback-requested", ok: false, detail: why });
+  deps.enqueue({
+    project: deps.projectName,
+    kind: "tier2",
+    body: [
+      `Fleet upgrade to omp-conductor@${request.version} did not verify (${why}) and is being rolled back by ${spawned.unit}.`,
+      "Dispatch stays paused until the rollback lands and a later tick verifies the old version.",
+      "",
+      `Journal: ${upgradeJournalPath(root)}`,
+    ].join("\n"),
+    at,
+    dedupeKey: `upgrade:${request.version}:rollback-requested`,
+  });
+  return { handled: "rollback-requested", version: request.version, detail: `${why}; rollback unit ${spawned.unit}` };
+}