omp-conductor 0.15.9 → 0.15.11

package/src/upgrade.ts

@@ -12,13 +12,28 @@ import {
   type FleetLayers,
 } from "./fleet.ts";
 import { livingDaemon, restartDaemon } from "./lifecycle.ts";
-import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, writeConfigRaw } from "./config.ts";
+import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";
 import {
   STAGED_SERVICE_NAME,
   planHostRuntime,
   writeHostRuntime,
 } from "./setup-host.ts";
+import {
+  appendJournal,
+  readUpgradeJournal,
+  upgradeJournalPath,
+  type UpgradeJournalEntry,
+} from "./upgrade-journal.ts";
+import {
+  enqueueUpgradeReport,
+  pendingUpgradeRequest,
+  reportProject,
+  requestProgression,
+  runCommand,
+  type PendingUpgradeRequest,
+  type UpgradeCommandResult,
+} from "./upgrade-verify.ts";
 
 const PACKAGE = "omp-conductor";
 const HERDR_PLUGIN = "herdr-conductor";
@@ -28,15 +43,17 @@ const DRAIN_POLL_MS = 5_000;
 const RECOVERY_POLL_MS = 2_000;
 const RECOVERY_ATTEMPTS = 30;
 
-export interface UpgradeCommandResult {
-  code: number;
-  stdout: string;
-  stderr: string;
-}
-
 export interface UpgradeOptions {
   version?: string;
   project?: string;
+  /**
+   * Run as the detached fleet installer (#486): journal every surface the
+   * transaction touches, skip the in-process verification and leave dispatch
+   * paused — the first tick of the post-restart daemon verifies version,
+   * /healthz, ticks, pane and doctor, and closes or rolls back on that
+   * evidence. Only the transient unit's `upgrade-install` entry sets this.
+   */
+  detached?: boolean;
 }
 
 export interface UpgradeResult {
@@ -79,25 +96,18 @@ export interface UpgradeDeps {
   sleep(ms: number): Promise<void>;
   env: NodeJS.ProcessEnv;
   log(message: string): void;
+  /**
+   * The durable journal sink for the detached fleet installer (#486). Wired
+   * to the state-dir journal by `upgrade-install`/`upgrade-rollback`; absent
+   * on the human `upgrade` path, whose own process is its witness. The engine
+   * journals each surface boundary so the post-restart tick can verify against
+   * a record it did not write.
+   */
+  journal?(entry: UpgradeJournalEntry): void;
 }
 
-async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
-  try {
-    const child = Bun.spawn([command, ...args], {
-      stdin: "ignore",
-      stdout: "pipe",
-      stderr: "pipe",
-      env: process.env,
-    });
-    const stdout = new Response(child.stdout).text();
-    const stderr = new Response(child.stderr).text();
-    const code = await child.exited;
-    return { code, stdout: await stdout, stderr: await stderr };
-  } catch (err) {
-    return { code: 127, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
-  }
-}
-
+/** One command run to completion with captured output. Exported so the
+ *  post-restart verifier and the rollback unit run processes the same way. */
 export const DEFAULT_DEPS: UpgradeDeps = {
   run: runCommand,
   snapshot: statusSnapshot,
@@ -576,7 +586,10 @@ async function upgradeBriefs(deps: UpgradeDeps, briefs: readonly ScopedBrief[]):
   }
 }
 
-async function rollbackUpgrade(
+/** Restore every surface the upgrade touched, in reverse, from the identities
+ *  captured before the first install. Exported so the detached rollback unit
+ *  can restore what the detached installer recorded (#486). */
+export async function rollbackUpgrade(
   deps: UpgradeDeps,
   previous: InstalledSurfaces,
   scope: UpgradeScope,
@@ -736,6 +749,10 @@ export async function upgradeConductor(
   overrides: Partial<UpgradeDeps> = {},
 ): Promise<UpgradeResult> {
   const deps: UpgradeDeps = { ...DEFAULT_DEPS, ...overrides };
+  const detached = options.detached === true;
+  const journal = (entry: Omit<UpgradeJournalEntry, "at">, at = new Date()): void => {
+    deps.journal?.({ ...entry, at: at.toISOString() });
+  };
   if (deps.env["HERDR_ENV"] !== undefined) {
     throw new Error("run omp-conductor upgrade from a shell outside the target Herdr session");
   }
@@ -748,6 +765,18 @@ export async function upgradeConductor(
   const release = parseRegistry(
     (await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
   );
+  if (detached) {
+    // The durable request: written before anything can pause, so a unit that
+    // dies after this line leaves a journal the returning process can act on
+    // (roll back, resume, or report) instead of a paused fleet with no record.
+    journal({
+      kind: "request",
+      ok: true,
+      version: release.version,
+      gitHead: release.gitHead,
+      unit: deps.env["OMP_CONDUCTOR_UNIT"],
+    });
+  }
   const initial = deps.layers(scope.pauseKey);
   const surfaces = await inspectSurfaces(deps);
   const briefs: ScopedBrief[] = scope.selectors.map((selector) => ({
@@ -756,6 +785,15 @@ export async function upgradeConductor(
   }));
   const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
   if (!installNeeded && briefs.every((b) => b.current)) {
+    if (detached) {
+      journal({
+        kind: "outcome",
+        phase: "already-current",
+        ok: true,
+        version: release.version,
+        detail: "all three surfaces and every brief were already pinned to the target release",
+      });
+    }
     return {
       previousVersion: surfaces.cliVersion,
       ...release,
@@ -796,20 +834,51 @@ export async function upgradeConductor(
   }
   if (initial.herdr === "unknown") throw new Error("cannot determine whether herdr-fleet.service is active");
 
+  if (detached) {
+    // The durable record the returning process rolls back or verifies against:
+    // what was installed, what dispatch state the fleet began in, which pause
+    // sentinel this transaction owns, and the config snapshot for the restore.
+    journal({
+      kind: "snapshot",
+      ok: true,
+      version: release.version,
+      gitHead: release.gitHead,
+      unit: deps.env["OMP_CONDUCTOR_UNIT"],
+      previous: {
+        cliVersion: surfaces.cliVersion,
+        ompVersion: surfaces.ompVersion,
+        herdrSource: surfaces.herdrSource,
+      },
+      ...(preUpgradeBackup === undefined ? {} : { configBackup: preUpgradeBackup }),
+      pauseKey: scope.pauseKey,
+      initialPaused: initial.paused,
+      initial: {
+        dispatch: initial.dispatch,
+        ticks: initial.ticks,
+        pane: initial.pane,
+        herdr: initial.herdr,
+      },
+      selectors: scope.selectors.map((s) => s ?? null),
+    });
+  }
+
   deps.log(`target release: omp-conductor@${release.version} (${release.gitHead})`);
   if (!initial.paused) {
     deps.log("safety: pausing new issue claims");
     deps.setPaused(true, scope.pauseKey);
   }
+  journal({ kind: "phase", phase: "paused", surface: "dispatch", ok: true, version: release.version });
   deps.log("drain: waiting for live omp worker sessions");
   try {
     await waitForDrain(deps, scope);
   } catch (err) {
+    journal({ kind: "phase", phase: "drain", ok: false, version: release.version, detail: err instanceof Error ? err.message : String(err) });
     const failure = err instanceof Error ? err.message : String(err);
     throw new Error(
       `upgrade failed while draining workers: ${failure}; no installation started; dispatch remains paused`,
     );
   }
+  journal({ kind: "phase", phase: "drain", ok: true, version: release.version });
   deps.log("drain: complete");
 
   let installTouched = false;
@@ -821,8 +890,10 @@ export async function upgradeConductor(
       installTouched = true;
       deps.log(`install 1/3: Bun-global omp-conductor CLI → ${release.version}`);
       await mustRun(deps, "bun", ["add", "-g", `${PACKAGE}@${release.version}`]);
+      journal({ kind: "phase", phase: "install", surface: "cli", ok: true, version: release.version, gitHead: release.gitHead });
       deps.log(`install 2/3: omp plugin omp-conductor → ${release.version}`);
       await mustRun(deps, "omp", ["plugin", "install", `${PACKAGE}@${release.version}`]);
+      journal({ kind: "phase", phase: "install", surface: "omp", ok: true, version: release.version, gitHead: release.gitHead });
       if (surfaces.herdrSource?.startsWith("local:")) {
         await mustRun(deps, "herdr", ["plugin", "unlink", HERDR_PLUGIN]);
       }
@@ -835,6 +906,7 @@ export async function upgradeConductor(
         release.gitHead,
         "--yes",
       ]);
+      journal({ kind: "phase", phase: "install", surface: "herdr", ok: true, version: release.version, gitHead: release.gitHead });
     } else {
       deps.log("install: CLI, omp plugin, and Herdr plugin already pinned to the target release");
     }
@@ -842,16 +914,40 @@ export async function upgradeConductor(
     deps.log("brief: refreshing managed ORCHESTRATOR.md floor while preserving POLICY.md");
     briefChanged = true;
     await upgradeBriefs(deps, briefs);
+    journal({ kind: "phase", phase: "brief", surface: "brief", ok: true, version: release.version });
 
     if (initial.herdr === "active") {
       herdrReloadStarted = true;
       deps.log("reload: restarting herdr-fleet.service and recovering the orchestrator pane");
       await mustRun(deps, "systemctl", ["restart", HERDR_UNIT]);
+      journal({ kind: "phase", phase: "restart", surface: "herdr", ok: true, version: release.version });
     }
     if (initial.dispatch !== "stopped") {
       daemonReloadStarted = true;
       deps.log("reload: restarting the omp-conductor dispatch daemon");
       await deps.restartDaemon();
+      journal({ kind: "phase", phase: "restart", surface: "daemon", ok: true, version: release.version });
+    }
+
+    if (detached) {
+      // The installing process does not declare its own success: it journals
+      // that the transaction reached the reloads and stops, leaving dispatch
+      // paused exactly where it was. The first tick of the post-restart daemon
+      // verifies against this journal and is the only witness that closes the
+      // transaction (#486).
+      journal({
+        kind: "phase",
+        phase: "awaiting-verification",
+        ok: true,
+        version: release.version,
+        detail: "installed; the first tick after the restart verifies and restores dispatch",
+      });
+      return {
+        previousVersion: surfaces.cliVersion,
+        ...release,
+        alreadyCurrent: false,
+        dispatch: initial.dispatch,
+      };
     }
 
     deps.log("verify 1/2: package identities, services, pane, ticks, and paused dispatch");
@@ -885,10 +981,21 @@ export async function upgradeConductor(
       deps.setPaused(true, scope.pauseKey);
     }
     const failure = err instanceof Error ? err.message : String(err);
+    journal({
+      kind: "phase",
+      phase: "failed",
+      ok: false,
+      version: release.version,
+      detail: failure,
+    });
     deps.log(`upgrade failed: ${failure}`);
+    const journalLog = (message: string): void => {
+      journal({ kind: "phase", phase: "rollback", ok: true, version: release.version, detail: message });
+      deps.log(message);
+    };
     try {
       await rollbackUpgrade(
-        deps,
+        { ...deps, log: journalLog },
         surfaces,
         scope,
         installTouched,
@@ -898,8 +1005,35 @@ export async function upgradeConductor(
         configBefore,
         preUpgradeBackup,
       );
+      journal({ kind: "outcome", phase: "rolled-back", ok: false, version: release.version, detail: failure });
+      enqueueUpgradeReport(
+        "tier2",
+        reportProject(scope.selectors),
+        `fleet upgrade to omp-conductor@${release.version} failed and was rolled back`,
+        [
+          `Reason: ${failure}`,
+          `Restored: cli=${surfaces.cliVersion}, omp=${surfaces.ompVersion ?? "missing"}, ` +
+            `herdr=${surfaces.herdrSource ?? "missing"}`,
+          ...(preUpgradeBackup === undefined ? [] : [`Pre-upgrade config snapshot: ${preUpgradeBackup}`]),
+          "Dispatch stays paused until the first tick after the restart records the rollback verified.",
+        ],
+        `upgrade:${release.version}:rolled-back`,
+      );
     } catch (rollbackErr) {
       const rollback = rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr);
+      journal({ kind: "outcome", phase: "rollback-failed", ok: false, version: release.version, detail: rollback });
+      enqueueUpgradeReport(
+        "tier2",
+        reportProject(scope.selectors),
+        `upgrade to omp-conductor@${release.version} failed and ITS ROLLBACK ALSO FAILED`,
+        [
+          `Failure: ${failure}`,
+          `Rollback failure: ${rollback}`,
+          "The fleet may be at mixed versions; do not resume dispatch.",
+          "Restore manually from the journal at " + upgradeJournalPath(stateDir()) + ".",
+        ],
+        `upgrade:${release.version}:rollback-failed`,
+      );
       throw new Error(
         `upgrade failed: ${failure}; rollback incomplete: ${rollback}; dispatch remains paused`,
       );
@@ -914,3 +1048,138 @@ export async function upgradeConductor(
     dispatch: initial.dispatch,
   };
 }
+
+// ---------------------------------------------------------------------------
+// The detached fleet installer (#486)
+// ---------------------------------------------------------------------------
+
+/**
+ * Run the upgrade transaction as the detached fleet installer: `upgrade`'s
+ * engine with a journal wired to the state-dir journal and the in-process
+ * verification handed to the first tick after the restart. This is the
+ * `upgrade-install` command the transient unit runs — the one process a
+ * `systemctl restart herdr-fleet.service` and a daemon restart both pass by.
+ */
+export async function runDetachedUpgrade(options: UpgradeOptions = {}): Promise<UpgradeResult> {
+  return upgradeConductor(
+    { ...options, detached: true },
+    {
+      journal: (entry) => appendJournal(stateDir(), entry),
+    },
+  );
+}
+
+// ---------------------------------------------------------------------------
+// The detached rollback unit (#486)
+// ---------------------------------------------------------------------------
+
+export interface RollbackFromJournalResult {
+  /** The version the rollback restored the fleet to. */
+  restoredVersion: string;
+  /** The version the failed upgrade had targeted. */
+  failedVersion: string;
+}
+
+/**
+ * Restore every surface the failed upgrade touched, from the journal the
+ * detached installer wrote. This is the `upgrade-rollback` command the
+ * verifier's transient unit runs — the same trust boundary as the install:
+ * the process that rolls back is not the process that failed, and it acts
+ * only on identities the journal recorded before the first install touched
+ * anything.
+ *
+ * Scoped to the newest request's progression: an older, already-closed
+ * upgrade in the same append-only journal neither supplies the snapshot to
+ * restore nor the phases that say which surfaces were touched, and its
+ * terminal outcome cannot refuse this rollback.
+ */
+export async function rollbackFromJournal(
+  overrides: Partial<UpgradeDeps> = {},
+  root = stateDir(),
+): Promise<RollbackFromJournalResult> {
+  const deps: UpgradeDeps = { ...DEFAULT_DEPS, ...overrides };
+  const entries = readUpgradeJournal(root);
+  const progression = requestProgression(entries);
+  const request = pendingUpgradeRequest(entries);
+  if (progression === undefined || request === undefined) {
+    throw new Error("no upgrade request is journaled; nothing to roll back");
+  }
+  const snapshot = progression.find((entry) => entry.kind === "snapshot");
+  if (snapshot === undefined || snapshot.previous === undefined) {
+    throw new Error(
+      `upgrade ${request.version} has no durable pre-install snapshot; refusing to guess what to restore`,
+    );
+  }
+  const newestOutcome = [...progression].reverse().find((entry) => entry.kind === "outcome");
+  if (newestOutcome !== undefined && newestOutcome.phase !== "rollback-requested") {
+    // Already terminal (verified, rolled back in-process, aborted). A second
+    // rollback would fight the newer outcome.
+    throw new Error(
+      `upgrade ${request.version} already closed as "${newestOutcome.phase}"; nothing to roll back`,
+    );
+  }
+
+  const phases = progression.filter((entry) => entry.kind === "phase");
+  const scope: UpgradeScope = {
+    selectors: request.selectors,
+    pauseKey: request.pauseKey,
+  };
+  const previous: InstalledSurfaces = {
+    cliVersion: snapshot.previous.cliVersion,
+    ompVersion: snapshot.previous.ompVersion,
+    herdrSource: snapshot.previous.herdrSource,
+  };
+  let configBefore: string | undefined;
+  if (request.configBackup !== undefined && existsSync(request.configBackup)) {
+    try {
+      configBefore = readFileSync(request.configBackup, "utf8");
+    } catch {
+      configBefore = undefined;
+    }
+  }
+  const journalLog = (message: string): void => {
+    appendJournal(root, {
+      at: new Date().toISOString(),
+      kind: "phase",
+      phase: "rollback",
+      ok: true,
+      version: request.version,
+      detail: message,
+    });
+    process.stdout.write(`${message}\n`);
+  };
+  await rollbackUpgrade(
+    { ...deps, log: journalLog },
+    previous,
+    scope,
+    phases.some((entry) => entry.phase === "install"),
+    phases.some((entry) => entry.phase === "brief"),
+    phases.some((entry) => entry.phase === "restart" && entry.surface === "herdr"),
+    phases.some((entry) => entry.phase === "restart" && entry.surface === "daemon"),
+    configBefore,
+    request.configBackup,
+  );
+
+  const restoredVersion = previous.cliVersion;
+  appendJournal(root, {
+    at: new Date().toISOString(),
+    kind: "outcome",
+    phase: "rolled-back",
+    ok: false,
+    version: request.version,
+    detail: `restored ${previous.cliVersion} from the pre-install snapshot`,
+  });
+  enqueueUpgradeReport(
+    "tier2",
+    reportProject(request.selectors),
+    `fleet restored to omp-conductor@${restoredVersion} after the failed upgrade to ${request.version}`,
+    [
+      "The transient rollback unit restored every surface from the durable snapshot;",
+      "dispatch stays paused until the first tick after these restarts verifies the old version.",
+      `Pre-upgrade config snapshot: ${request.configBackup ?? "none recorded"}`,
+    ],
+    `upgrade:${request.version}:rolled-back`,
+  );
+  return { restoredVersion, failedVersion: request.version };
+}
+

package/src/verbs/actions.ts

@@ -15,9 +15,13 @@
  */
 
 import { credentialedEnv, openRunPr, pushRunBranch, repoSlugFor, scrubUserinfo } from "../gitops.ts";
+import { livingDaemon } from "../lifecycle.ts";
+import { stateDir } from "../config.ts";
 import { ensureMirror } from "../worktree.ts";
+import { launchTransientUnit, readUpgradeJournal } from "../upgrade-journal.ts";
+import { classifyUpgrade } from "../upgrade-verify.ts";
 import type { ProjectConfig, ReleaseShape, RepoTarget } from "../types.ts";
-import type { ActionOutcome, ReleaseExecution, VerbActions } from "./server.ts";
+import type { ActionOutcome, InstallExecution, ReleaseExecution, VerbActions } from "./server.ts";
 
 /**
  * What this daemon can actually cut.
@@ -779,5 +783,73 @@ export function githubVerbActions(
           }
         : failed(pushed.result, pushed.argv);
     },
+
+    // The fleet-installs-itself request (#486). Two gates, then a spawn:
+    //
+    // 1. `npm view` must actually expose the version with a full gitHead —
+    //    "install it" may only name what npm can serve. Refused with npm's
+    //    own detail so the orchestrator learns why, not that something was
+    //    wrong.
+    // 2. A live daemon must exist to verify the install on its first tick
+    //    after the restart. A stopped daemon can never come back to do that,
+    //    so the request is refused and the operator is pointed at the plain
+    //    `upgrade` command, whose own process is its witness.
+    install: async (execution: InstallExecution): Promise<ActionOutcome> => {
+      if (livingDaemon() === undefined) {
+        return {
+          ok: false,
+          stderr:
+            "the dispatch daemon is not running, so nobody can verify this install after the restart " +
+            "it performs. Run `omp-conductor upgrade` from a shell instead.",
+        };
+      }
+      // One install at a time: the journal's newest request is either closed or
+      // in flight, and a second request while one is in flight would race it
+      // for the same restarts. The refusal names the state so the caller knows
+      // to wait for the verifier's outcome instead of retrying blind.
+      const pending = classifyUpgrade(readUpgradeJournal(stateDir()));
+      if (pending.kind !== "none" && pending.kind !== "closed") {
+        return {
+          ok: false,
+          stderr:
+            `an install is already in flight (journal verdict: ${pending.kind}); ` +
+            "the fleet verifies or rolls it back before accepting another request.",
+        };
+      }
+      const looked = await run(
+        ["npm", "view", `omp-conductor@${execution.version}`, "version", "gitHead", "--json"],
+        { env: env() },
+      );
+      if (!looked.ok) {
+        const detail = looked.stderr.trim() || looked.stdout.trim() || "the npm view command failed";
+        return {
+          ok: false,
+          stderr:
+            `npm does not expose omp-conductor@${execution.version}: ${detail}. ` +
+            "Only a published release with a full gitHead can be installed; the request was not started.",
+        };
+      }
+      const metadata = objectFromJson(looked.stdout);
+      const published = metadata === undefined ? undefined : String(metadata["version"] ?? "");
+      const gitHead = metadata === undefined ? undefined : String(metadata["gitHead"] ?? "");
+      if (published !== execution.version || !/^[0-9a-f]{40}$/i.test(gitHead ?? "")) {
+        return {
+          ok: false,
+          stderr:
+            `npm does not expose omp-conductor@${execution.version} with a full gitHead ` +
+            `(${published === undefined ? "unreadable metadata" : `saw ${published}`}); the request was not started.`,
+        };
+      }
+      const launched = await launchTransientUnit(
+        async (argv) => {
+          const ran = await run([...argv] as string[], { env: env() });
+          return { code: ran.ok ? 0 : 1, stdout: ran.stdout, stderr: ran.stderr };
+        },
+        env(),
+        { kind: "install", version: execution.version },
+      );
+      if (!launched.ok) return { ok: false, stderr: launched.stderr };
+      return { ok: true, detail: launched.unit };
+    },
   };
 }

package/src/verbs/protocol.ts

@@ -288,15 +288,40 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
       `release authority is the orchestrator's, and this is a ${role} session. ` +
       "A worker session holds no release grant whatever the config says.",
   },
+  conductor_install: {
+    name: "conductor_install",
+    mutating: true,
+    allowedRoles: ["orchestrator"],
+    description:
+      "Request the fleet upgrade itself to a published omp-conductor release (#486). The install " +
+      "shape gates it like a release shape and stays with the human by default; with the grant, " +
+      "the daemon validates the version against npm and starts a detached transient unit that " +
+      "pauses, drains, installs the CLI/omp plugin/Herdr plugin and reloads — outside this session " +
+      "and the daemon. The unit never declares its own success: the first tick after the restart " +
+      "verifies version, /healthz, ticks, pane and doctor, and reports through the durable outbox.",
+    args: {
+      version: {
+        type: "string",
+        required: true,
+        description:
+          "The exact published semver to install (e.g. 0.15.11). The request is refused when npm " +
+          "does not expose it with a full gitHead.",
+      },
+      rationale: RATIONALE_ARG,
+    },
+    roleRefusalText: (role) =>
+      `install authority is the orchestrator's, and this is a ${role} session. ` +
+      "A worker session holds no install grant whatever the config says.",
+  },
   conductor_pr_status: {
     name: "conductor_pr_status",
     mutating: false,
     allowedRoles: ["worker", "orchestrator"],
     description:
-      "Read one pull request's live state and current head: is it open and are its checks green. " +
-      "Optionally compare that head with one the caller already observed. This is the daemon's own " +
-      "merge-gate verdict, not a summary of it. A poll, not a watcher — call it again after waiting. " +
-      "A worker may omit prUrl and gets its own run's.",
+      "Read one pull request's live state and current head — open, merged, or closed — and the checks " +
+      "available for it (checks are unavailable for a merged or closed PR). Optionally compare that head " +
+      "with one the caller already observed. This is the daemon's own merge-gate verdict, not a summary of it. " +
+      "A poll, not a watcher — call it again after waiting. A worker may omit prUrl and gets its own run's.",
     args: {
       prUrl: {
         type: "string",