omp-conductor 0.15.9 → 0.15.11

package/src/upgrade-journal.ts

@@ -0,0 +1,220 @@
+/**
+ * The durable journal for fleet-initiated upgrades (#486).
+ *
+ * `omp-conductor upgrade` has always been a transaction *inside* one process:
+ * it paused, drained, installed, reloaded and verified, and the process that
+ * verified was the process that restarted the fleet. The fleet-installs-itself
+ * path replaces that wrapper, not the transaction: a detached transient unit
+ * performs the install while this module records every surface it touches, and
+ * the first tick of the post-restart daemon verifies against the journal —
+ * a process that did not run the install, which is the only witness worth
+ * trusting after the restarts.
+ *
+ * Everything here is a leaf on purpose: the verb server, the daemon's
+ * privileged actions and the upgrade engine all need the same journal and the
+ * same transient-unit naming, and importing `upgrade.ts` from both verbs and
+ * daemon would close import cycles that have no business existing. The module
+ * holds no state of its own; the journal is append-only JSONL under the
+ * conductor state directory, so a process that died mid-write leaves every
+ * line before the torn one readable.
+ */
+
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+import { stateDir } from "./config.ts";
+
+/**
+ * Where the fleet's upgrade journal lives: one JSONL file per host, because
+ * the transaction it records is host-wide — one daemon serves every configured
+ * project, and the install replaces every surface at once.
+ */
+export const UPGRADE_JOURNAL_FILE = "upgrade-journal.jsonl";
+
+/** The packages this fleet never leaves on mixed versions. */
+export interface JournalSurfaces {
+  cliVersion: string;
+  ompVersion?: string;
+  herdrSource?: string;
+}
+
+/** What the fleet looked like when the install began, for delta verification. */
+export interface JournalFleetState {
+  dispatch: string;
+  ticks: string;
+  pane: string;
+  herdr: string;
+}
+
+/** One named verification check, recorded so the outcome is evidence, not prose. */
+export interface UpgradeCheck {
+  name: string;
+  ok: boolean;
+  detail?: string;
+}
+
+export interface UpgradeJournalEntry {
+  /** ISO timestamp of the write. */
+  at: string;
+  kind: "request" | "snapshot" | "phase" | "outcome";
+  /** Always present for request/outcome; phases carry them for searchability. */
+  version?: string;
+  gitHead?: string;
+  /** The transient unit performing the install, when it named itself. */
+  unit?: string;
+  /** The step name: paused, drain, install, brief, restart, awaiting-verification, … */
+  phase?: string;
+  /** The surface one phase touched: cli/omp/herdr/brief/herdr-service/daemon/dispatch. */
+  surface?: string;
+  ok: boolean;
+  detail?: string;
+  /** The durable pre-install identities, for the process that must roll back. */
+  previous?: InstallSnapshotSurfaces;
+  configBackup?: string;
+  /** The pause sentinel the install owns, for the verifier that must lift it. */
+  pauseKey?: string;
+  /** Whether dispatch was already paused when the request began. */
+  initialPaused?: boolean;
+  /** The fleet layers the install began under, for delta verification. */
+  initial?: JournalFleetState;
+  /** The projects the install covers, in the form the upgrade engine uses. */
+  selectors?: readonly (string | null)[];
+  /** Verification evidence when the post-restart tick closes the transaction. */
+  checks?: readonly UpgradeCheck[];
+}
+
+/** The installed identities a rollback must restore, verbatim. */
+export interface InstallSnapshotSurfaces {
+  cliVersion: string;
+  ompVersion?: string;
+  herdrSource?: string;
+}
+
+export function upgradeJournalPath(root = stateDir()): string {
+  return join(root, UPGRADE_JOURNAL_FILE);
+}
+
+export function readUpgradeJournal(root = stateDir()): UpgradeJournalEntry[] {
+  const path = upgradeJournalPath(root);
+  if (!existsSync(path)) return [];
+  let text: string;
+  try {
+    text = readFileSync(path, "utf8");
+  } catch {
+    // An unreadable journal is the same answer as an absent one for the read
+    // side; the writer keeps appending and the operator can recover the file.
+    return [];
+  }
+  const entries: UpgradeJournalEntry[] = [];
+  for (const line of text.split("\n")) {
+    if (line.length === 0) continue;
+    try {
+      const entry = JSON.parse(line) as UpgradeJournalEntry;
+      if (entry !== null && typeof entry === "object") entries.push(entry);
+    } catch {
+      // One torn line (the process died mid-append) does not hide the earlier
+      // evidence; it is exactly the "interrupted transaction" the journal was
+      // built to diagnose.
+    }
+  }
+  return entries;
+}
+
+/**
+ * Append one line. Each write is one `appendFileSync`, so a crash cannot
+ * interleave two entries; callers order their own phases.
+ */
+export function appendJournal(root: string, entry: UpgradeJournalEntry): void {
+  mkdirSync(root, { recursive: true });
+  appendFileSync(
+    join(root, UPGRADE_JOURNAL_FILE),
+    `${JSON.stringify(entry)}\n`,
+    { mode: 0o600 },
+  );
+}
+
+/**
+ * The transient systemd unit that performs an install or its rollback. The
+ * unit lives in the system manager, not in the pane's process tree and not in
+ * the daemon's cgroup, so `systemctl restart herdr-fleet.service` and the
+ * daemon restart both pass it by. The version is slugged into the unit name
+ * because the unit names the transaction; a second request for the same
+ * version collides with the first on purpose (see the launch guard).
+ */
+export function transientUnitName(version: string, kind: "install" | "rollback"): string {
+  const slug = version.replace(/[^a-zA-Z0-9_.-]/g, "-");
+  return `omp-conductor-upgrade-${kind}-${slug}`;
+}
+
+/**
+ * The explicit PATH a detached unit must carry. On this fleet's host `omp`,
+ * `herdr` and `omp-conductor` are absent from the non-interactive ssh PATH,
+ * and a transient unit starts with systemd's own minimal PATH — so the
+ * executor prepends the two well-known runtime dirs and keeps whatever the
+ * daemon itself ran with. Deduplicated so a dir already on the path is not
+ * listed twice.
+ */
+const EXECUTOR_PATH_DIRS = ["/root/.bun/bin", "/root/.local/bin"];
+
+export function executorPath(env: NodeJS.ProcessEnv): string {
+  const existing = typeof env["PATH"] === "string" ? env["PATH"] : "";
+  const parts = [...EXECUTOR_PATH_DIRS, ...existing.split(":").filter((part) => part.length > 0)];
+  return [...new Set(parts)].join(":");
+}
+
+export type UnitSpawnResult = { code: number; stdout: string; stderr: string };
+export type UnitSpawnFn = (argv: readonly string[]) => Promise<UnitSpawnResult>;
+
+export type UnitCommand =
+  | { kind: "install"; version: string }
+  | { kind: "rollback"; version: string };
+
+export interface LaunchedUnit {
+  ok: true;
+  unit: string;
+}
+export interface UnitLaunchRefusal {
+  ok: false;
+  stderr: string;
+}
+
+/**
+ * Start one detached transient unit that performs an install or a rollback.
+ *
+ * The environment is explicit and minimal on purpose: `PATH` is composed by
+ * {@link executorPath} so the unit can find `omp`, `herdr` and
+ * `omp-conductor`, and the requesting context's `HERDR_SESSION` (if any) is
+ * passed through so the fleet the unit talks to is the fleet the request
+ * meant. The unit is `--collect`ed so a finished unit leaves no failed-unit
+ * litter behind. The spawning process keeps the unit name; the unit is told
+ * through `OMP_CONDUCTOR_UNIT` so its journal records which unit ran.
+ */
+export async function launchTransientUnit(
+  spawn: UnitSpawnFn,
+  env: NodeJS.ProcessEnv,
+  command: UnitCommand,
+): Promise<LaunchedUnit | UnitLaunchRefusal> {
+  const unit = transientUnitName(command.version, command.kind);
+  const setenv = [`PATH=${executorPath(env)}`, `OMP_CONDUCTOR_UNIT=${unit}`];
+  const session = env["HERDR_SESSION"];
+  if (session !== undefined && session.length > 0) setenv.push(`HERDR_SESSION=${session}`);
+  const argv = [
+    "systemd-run",
+    `--unit=${unit}`,
+    "--collect",
+    ...setenv.map((pair) => `--setenv=${pair}`),
+    "omp-conductor",
+    command.kind === "install" ? "upgrade-install" : "upgrade-rollback",
+    ...(command.kind === "install" ? ["--to", command.version] : []),
+  ];
+  const ran = await spawn(argv);
+  if (ran.code !== 0) {
+    const detail = ran.stderr.trim() || ran.stdout.trim() || `exit ${ran.code}`;
+    return {
+      ok: false,
+      stderr:
+        `could not start ${unit} through systemd-run: ${detail}. ` +
+        "Nothing was installed and dispatch was not touched; run `omp-conductor upgrade` from a shell instead.",
+    };
+  }
+  return { ok: true, unit };
+}