omp-conductor 0.3.18 → 0.3.20

package/src/upgrade.ts

@@ -0,0 +1,537 @@
+import { readFileSync } from "node:fs";
+import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
+import { setPaused, statusSnapshot } from "./daemon.ts";
+import { fleetLayers, type DispatchLayer, type FleetLayers } from "./fleet.ts";
+import { livingDaemon, restartDaemon } from "./lifecycle.ts";
+import { findProject, loadConfig } from "./config.ts";
+import { renderBriefForProject } from "./setup.ts";
+
+const PACKAGE = "omp-conductor";
+const HERDR_PLUGIN = "herdr-conductor";
+const HERDR_SOURCE = "TerrifiedBug/conductor/herdr";
+const HERDR_UNIT = "herdr-fleet.service";
+const DRAIN_POLL_MS = 5_000;
+const RECOVERY_POLL_MS = 2_000;
+const RECOVERY_ATTEMPTS = 30;
+
+export interface UpgradeCommandResult {
+  code: number;
+  stdout: string;
+  stderr: string;
+}
+
+export interface UpgradeOptions {
+  version?: string;
+  project?: string;
+}
+
+export interface UpgradeResult {
+  previousVersion: string;
+  version: string;
+  gitHead: string;
+  alreadyCurrent: boolean;
+  dispatch: DispatchLayer;
+}
+
+export interface UpgradeDeps {
+  run(command: string, args: readonly string[]): Promise<UpgradeCommandResult>;
+  snapshot(project?: string): { liveWorkers: number };
+  layers(project?: string): FleetLayers;
+  brief(project?: string): { kind: BriefLayout["kind"]; current: boolean };
+  daemonIdentity(): { running: boolean; project?: string };
+  setPaused(value: boolean): void;
+  restartDaemon(): Promise<void>;
+  sleep(ms: number): Promise<void>;
+  env: NodeJS.ProcessEnv;
+  log(message: string): void;
+}
+
+async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
+  try {
+    const child = Bun.spawn([command, ...args], {
+      stdin: "ignore",
+      stdout: "pipe",
+      stderr: "pipe",
+      env: process.env,
+    });
+    const stdout = new Response(child.stdout).text();
+    const stderr = new Response(child.stderr).text();
+    const code = await child.exited;
+    return { code, stdout: await stdout, stderr: await stderr };
+  } catch (err) {
+    return { code: 127, stdout: "", stderr: err instanceof Error ? err.message : String(err) };
+  }
+}
+
+const DEFAULT_DEPS: UpgradeDeps = {
+  run: runCommand,
+  snapshot: statusSnapshot,
+  layers: fleetLayers,
+  brief: (projectName) => {
+    const project = findProject(loadConfig(), projectName);
+    const layout = inspectBriefLayout(project.workspaceRoot, renderBriefForProject(project));
+    if (layout.kind !== "overlay") return { kind: layout.kind, current: false };
+    const policy = readFileSync(layout.policyPath, "utf8");
+    const live = readFileSync(layout.orchestratorPath, "utf8");
+    return { kind: layout.kind, current: live === renderBriefForProject(project, policy) };
+  },
+  daemonIdentity: () => {
+    const daemon = livingDaemon();
+    return daemon === undefined ? { running: false } : { running: true, project: daemon.project };
+  },
+  setPaused,
+  restartDaemon: async () => {
+    await restartDaemon({});
+  },
+  sleep: Bun.sleep,
+  env: process.env,
+  log: (message) => process.stdout.write(`${message}\n`),
+};
+
+function commandLine(command: string, args: readonly string[]): string {
+  return [command, ...args].join(" ");
+}
+
+async function mustRun(
+  deps: UpgradeDeps,
+  command: string,
+  args: readonly string[],
+): Promise<UpgradeCommandResult> {
+  const ran = await deps.run(command, args);
+  if (ran.code === 0) return ran;
+  const detail = ran.stderr.trim() || ran.stdout.trim() || `exit ${ran.code}`;
+  throw new Error(`${commandLine(command, args)} failed: ${detail}`);
+}
+
+function parseRegistry(raw: string): { version: string; gitHead: string } {
+  let value: unknown;
+  try {
+    value = JSON.parse(raw);
+  } catch {
+    throw new Error("npm returned invalid release metadata");
+  }
+  if (value === null || typeof value !== "object" || Array.isArray(value)) {
+    throw new Error("npm returned invalid release metadata");
+  }
+  const version = Reflect.get(value, "version");
+  const gitHead = Reflect.get(value, "gitHead");
+  if (typeof version !== "string" || version.length === 0) {
+    throw new Error("npm release metadata has no version");
+  }
+  if (typeof gitHead !== "string" || !/^[0-9a-f]{40}$/i.test(gitHead)) {
+    throw new Error(`npm release ${version} has no full gitHead`);
+  }
+  return { version, gitHead };
+}
+
+function ompPluginVersion(raw: string): string | undefined {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error("omp plugin list returned invalid JSON");
+  }
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) {
+    throw new Error("omp plugin list returned an invalid inventory");
+  }
+  const npm = Reflect.get(parsed, "npm");
+  if (!Array.isArray(npm)) throw new Error("omp plugin list returned no npm inventory");
+  const plugin = npm.find(
+    (entry) => entry !== null && typeof entry === "object" && Reflect.get(entry, "name") === PACKAGE,
+  );
+  if (plugin === undefined) return undefined;
+  const version = Reflect.get(plugin, "version");
+  if (typeof version !== "string" || version.length === 0) {
+    throw new Error("installed omp-conductor plugin has no version");
+  }
+  return version;
+}
+
+function herdrPluginSource(raw: string): string | undefined {
+  const line = raw.split("\n").find((candidate) => /^- herdr-conductor\b/.test(candidate.trim()));
+  if (line === undefined) return undefined;
+  const source = line.match(/\[([^\]]+)\]\s*$/)?.[1];
+  if (source === undefined) {
+    throw new Error("cannot determine installed Herdr plugin source");
+  }
+  return source;
+}
+
+interface InstalledSurfaces {
+  cliVersion: string;
+  ompVersion?: string;
+  herdrSource?: string;
+}
+
+async function inspectSurfaces(deps: UpgradeDeps): Promise<InstalledSurfaces> {
+  const session = deps.env["HERDR_SESSION"] || "fleet";
+  const [cli, omp, herdr] = await Promise.all([
+    mustRun(deps, "omp-conductor", ["--version"]),
+    mustRun(deps, "omp", ["plugin", "list", "--json"]),
+    mustRun(deps, "herdr", ["--session", session, "plugin", "list"]),
+  ]);
+  const cliVersion = cli.stdout.trim();
+  if (cliVersion.length === 0) throw new Error("installed omp-conductor CLI has no version");
+  return {
+    cliVersion,
+    ompVersion: ompPluginVersion(omp.stdout),
+    herdrSource: herdrPluginSource(herdr.stdout),
+  };
+}
+
+function expectedHerdrSource(gitHead: string): string {
+  return `github:${HERDR_SOURCE}@${gitHead}`;
+}
+
+function surfacesCurrent(
+  surfaces: Awaited<ReturnType<typeof inspectSurfaces>>,
+  version: string,
+  gitHead: string,
+): boolean {
+  return (
+    surfaces.cliVersion === version &&
+    surfaces.ompVersion === version &&
+    surfaces.herdrSource === expectedHerdrSource(gitHead)
+  );
+}
+
+function previousHerdrInstall(source: string): readonly [string, readonly string[]] | undefined {
+  if (source.startsWith("local:")) {
+    return ["herdr", ["plugin", "link", source.slice("local:".length), "--enabled"]];
+  }
+  if (!source.startsWith("github:")) return undefined;
+  const at = source.lastIndexOf("@");
+  if (at < "github:".length) return undefined;
+  return [
+    "herdr",
+    ["plugin", "install", source.slice("github:".length, at), "--ref", source.slice(at + 1), "--yes"],
+  ];
+}
+
+async function waitForDrain(deps: UpgradeDeps, project?: string): Promise<void> {
+  let last = -1;
+  while (true) {
+    const workers = deps.snapshot(project).liveWorkers;
+    if (workers === 0) return;
+    if (workers !== last) deps.log(`waiting for ${workers} live worker(s) to finish`);
+    last = workers;
+    await deps.sleep(DRAIN_POLL_MS);
+  }
+}
+
+function recoveryProblem(
+  layers: FleetLayers,
+  initial: FleetLayers,
+  liveWorkers: number,
+): string | undefined {
+  if (layers.ticks !== initial.ticks) return `ticks changed from ${initial.ticks} to ${layers.ticks}`;
+  if (layers.recovery !== initial.recovery) {
+    return `recovery changed from ${initial.recovery} to ${layers.recovery}`;
+  }
+  if (initial.herdr === "active" && layers.herdr !== "active") return `Herdr is ${layers.herdr}`;
+  if (initial.herdr === "active" && initial.recovery === "clear" && layers.pane !== "live") {
+    return `orchestrator pane is ${layers.pane}`;
+  }
+  if (initial.dispatch !== "stopped" && !layers.daemon.running) return "dispatch daemon is not running";
+  if (initial.dispatch === "stopped" && layers.daemon.running) return "stopped dispatch daemon was started";
+  if (initial.dispatch !== "stopped" && !layers.paused) return "dispatch resumed before verification";
+  if (liveWorkers !== 0) return `${liveWorkers} worker(s) appeared during verification`;
+  return undefined;
+}
+
+async function waitForRecovery(
+  deps: UpgradeDeps,
+  initial: FleetLayers,
+  project?: string,
+): Promise<void> {
+  let problem = "recovery did not settle";
+  for (let attempt = 0; attempt < RECOVERY_ATTEMPTS; attempt += 1) {
+    problem = recoveryProblem(deps.layers(project), initial, deps.snapshot(project).liveWorkers) ?? "";
+    if (problem === "") return;
+    await deps.sleep(RECOVERY_POLL_MS);
+  }
+  throw new Error(`upgrade verification failed: ${problem}`);
+}
+
+async function upgradeBrief(
+  deps: UpgradeDeps,
+  kind: BriefLayout["kind"],
+  project?: string,
+): Promise<void> {
+  const selected = project === undefined ? [] : ["--project", project];
+  if (kind === "legacy-handwritten") {
+    await mustRun(deps, "omp-conductor", ["brief-upgrade", "--retrofit", "--apply", ...selected]);
+  }
+  await mustRun(deps, "omp-conductor", ["brief-upgrade", "--migrate", "--apply", ...selected]);
+}
+
+async function rollbackUpgrade(
+  deps: UpgradeDeps,
+  previous: InstalledSurfaces,
+  project: string | undefined,
+  installTouched: boolean,
+  briefChanged: boolean,
+  herdrReloadStarted: boolean,
+  daemonReloadStarted: boolean,
+): Promise<void> {
+  const failures: string[] = [];
+  const restore = async (label: string, command: string, args: readonly string[]): Promise<void> => {
+    deps.log(`rollback: ${label}`);
+    try {
+      await mustRun(deps, command, args);
+    } catch (err) {
+      failures.push(err instanceof Error ? err.message : String(err));
+    }
+  };
+  const removeIfPresent = async (
+    label: string,
+    command: string,
+    args: readonly string[],
+  ): Promise<void> => {
+    deps.log(`rollback: ${label}`);
+    try {
+      await deps.run(command, args);
+    } catch {
+      // Exact post-rollback identity verification decides whether absence is safe.
+    }
+  };
+
+  if (installTouched) {
+    await restore("Bun-global omp-conductor CLI", "bun", [
+      "add",
+      "-g",
+      `${PACKAGE}@${previous.cliVersion}`,
+    ]);
+    if (previous.ompVersion === undefined) {
+      await removeIfPresent("remove newly installed omp plugin", "omp", [
+        "plugin",
+        "uninstall",
+        PACKAGE,
+      ]);
+    } else {
+      await restore("omp plugin omp-conductor", "omp", [
+        "plugin",
+        "install",
+        `${PACKAGE}@${previous.ompVersion}`,
+      ]);
+    }
+
+    if (previous.herdrSource === undefined) {
+      await removeIfPresent("remove newly installed Herdr plugin", "herdr", [
+        "plugin",
+        "uninstall",
+        HERDR_PLUGIN,
+      ]);
+    } else {
+      const install = previousHerdrInstall(previous.herdrSource);
+      if (install === undefined) {
+        failures.push(`cannot restore unknown Herdr source ${previous.herdrSource}`);
+      } else {
+        if (previous.herdrSource.startsWith("local:")) {
+          await removeIfPresent("remove managed Herdr plugin", "herdr", [
+            "plugin",
+            "uninstall",
+            HERDR_PLUGIN,
+          ]);
+        }
+        await restore("Herdr plugin herdr-conductor", install[0], install[1]);
+      }
+    }
+
+  }
+  if (briefChanged) {
+    try {
+      deps.log("rollback: ORCHESTRATOR.md package floor");
+      await upgradeBrief(deps, "overlay", project);
+    } catch (err) {
+      failures.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+
+  if (herdrReloadStarted) {
+    await restore("restart herdr-fleet.service", "systemctl", ["restart", HERDR_UNIT]);
+  }
+  if (daemonReloadStarted) {
+    try {
+      deps.log("rollback: restart omp-conductor dispatch daemon");
+      await deps.restartDaemon();
+    } catch (err) {
+      failures.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+
+  if (installTouched) {
+    try {
+      const restored = await inspectSurfaces(deps);
+      if (
+        restored.cliVersion !== previous.cliVersion ||
+        restored.ompVersion !== previous.ompVersion ||
+        restored.herdrSource !== previous.herdrSource
+      ) {
+        failures.push(
+          `restored identities differ: cli=${restored.cliVersion}, omp=${restored.ompVersion ?? "missing"}, ` +
+            `herdr=${restored.herdrSource ?? "missing"}`,
+        );
+      }
+    } catch (err) {
+      failures.push(err instanceof Error ? err.message : String(err));
+    }
+  }
+
+  if (failures.length > 0) throw new Error(failures.join("; "));
+}
+
+export async function upgradeConductor(
+  options: UpgradeOptions = {},
+  overrides: Partial<UpgradeDeps> = {},
+): Promise<UpgradeResult> {
+  const deps: UpgradeDeps = { ...DEFAULT_DEPS, ...overrides };
+  if (deps.env["HERDR_ENV"] !== undefined) {
+    throw new Error("run omp-conductor upgrade from a shell outside the target Herdr session");
+  }
+
+  const requested = options.version === undefined ? `${PACKAGE}@latest` : `${PACKAGE}@${options.version}`;
+  const daemon = deps.daemonIdentity();
+  if (daemon.running && options.project !== undefined && daemon.project !== options.project) {
+    const active = daemon.project === undefined ? "an unrecorded project" : daemon.project;
+    throw new Error(
+      `active daemon serves ${active}; refusing to upgrade --project ${options.project} while it is running`,
+    );
+  }
+  const project = options.project ?? daemon.project;
+
+  const release = parseRegistry(
+    (await mustRun(deps, "npm", ["view", requested, "version", "gitHead", "--json"])).stdout,
+  );
+  const initial = deps.layers(project);
+  const surfaces = await inspectSurfaces(deps);
+  const brief = deps.brief(project);
+  const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
+  if (!installNeeded && brief.current) {
+    return {
+      previousVersion: surfaces.cliVersion,
+      ...release,
+      alreadyCurrent: true,
+      dispatch: initial.dispatch,
+    };
+  }
+
+  if (brief.kind === "missing") throw new Error("no ORCHESTRATOR.md exists for the configured project");
+  if (initial.herdr === "unknown") throw new Error("cannot determine whether herdr-fleet.service is active");
+
+  deps.log(`target release: omp-conductor@${release.version} (${release.gitHead})`);
+  if (!initial.paused) {
+    deps.log("safety: pausing new issue claims");
+    deps.setPaused(true);
+  }
+  deps.log("drain: waiting for live omp worker sessions");
+  try {
+    await waitForDrain(deps, project);
+  } catch (err) {
+    const failure = err instanceof Error ? err.message : String(err);
+    throw new Error(
+      `upgrade failed while draining workers: ${failure}; no installation started; dispatch remains paused`,
+    );
+  }
+  deps.log("drain: complete");
+
+  let installTouched = false;
+  let briefChanged = false;
+  let herdrReloadStarted = false;
+  let daemonReloadStarted = false;
+  try {
+    if (installNeeded) {
+      installTouched = true;
+      deps.log(`install 1/3: Bun-global omp-conductor CLI → ${release.version}`);
+      await mustRun(deps, "bun", ["add", "-g", `${PACKAGE}@${release.version}`]);
+      deps.log(`install 2/3: omp plugin omp-conductor → ${release.version}`);
+      await mustRun(deps, "omp", ["plugin", "install", `${PACKAGE}@${release.version}`]);
+      if (surfaces.herdrSource?.startsWith("local:")) {
+        await mustRun(deps, "herdr", ["plugin", "unlink", HERDR_PLUGIN]);
+      }
+      deps.log(`install 3/3: Herdr plugin herdr-conductor → ${release.gitHead}`);
+      await mustRun(deps, "herdr", [
+        "plugin",
+        "install",
+        HERDR_SOURCE,
+        "--ref",
+        release.gitHead,
+        "--yes",
+      ]);
+    } else {
+      deps.log("install: CLI, omp plugin, and Herdr plugin already pinned to the target release");
+    }
+
+    deps.log("brief: refreshing managed ORCHESTRATOR.md floor while preserving POLICY.md");
+    briefChanged = true;
+    await upgradeBrief(deps, brief.kind, project);
+
+    if (initial.herdr === "active") {
+      herdrReloadStarted = true;
+      deps.log("reload: restarting herdr-fleet.service and recovering the orchestrator pane");
+      await mustRun(deps, "systemctl", ["restart", HERDR_UNIT]);
+    }
+    if (initial.dispatch !== "stopped") {
+      daemonReloadStarted = true;
+      deps.log("reload: restarting the omp-conductor dispatch daemon");
+      await deps.restartDaemon();
+    }
+
+    deps.log("verify 1/2: package identities, services, pane, ticks, and paused dispatch");
+    const installed = await inspectSurfaces(deps);
+    if (!surfacesCurrent(installed, release.version, release.gitHead)) {
+      throw new Error(
+        `installed identities differ: cli=${installed.cliVersion}, omp=${installed.ompVersion ?? "missing"}, ` +
+          `herdr=${installed.herdrSource ?? "missing"}`,
+      );
+    }
+    if (initial.herdr === "active") {
+      const active = await mustRun(deps, "systemctl", ["is-active", HERDR_UNIT]);
+      if (active.stdout.trim() !== "active") throw new Error(`${HERDR_UNIT} is not active after restart`);
+    }
+    await waitForRecovery(deps, initial, project);
+    deps.log("verify 2/2: recovered fleet remains stable");
+    await deps.sleep(1_000);
+    await waitForRecovery(deps, initial, project);
+
+    if (!initial.paused) deps.setPaused(false);
+    const restoredDispatch = deps.layers(project).dispatch;
+    if (restoredDispatch !== initial.dispatch) {
+      throw new Error(
+        `dispatch state is ${restoredDispatch}; expected to restore ${initial.dispatch}`,
+      );
+    }
+  } catch (err) {
+    try {
+      if (!deps.layers(project).paused) deps.setPaused(true);
+    } catch {
+      deps.setPaused(true);
+    }
+    const failure = err instanceof Error ? err.message : String(err);
+    deps.log(`upgrade failed: ${failure}`);
+    try {
+      await rollbackUpgrade(
+        deps,
+        surfaces,
+        project,
+        installTouched,
+        briefChanged,
+        herdrReloadStarted,
+        daemonReloadStarted,
+      );
+    } catch (rollbackErr) {
+      const rollback = rollbackErr instanceof Error ? rollbackErr.message : String(rollbackErr);
+      throw new Error(
+        `upgrade failed: ${failure}; rollback incomplete: ${rollback}; dispatch remains paused`,
+      );
+    }
+    throw new Error(`upgrade failed: ${failure}; previous installation restored; dispatch remains paused`);
+  }
+
+  return {
+    previousVersion: surfaces.cliVersion,
+    ...release,
+    alreadyCurrent: false,
+    dispatch: initial.dispatch,
+  };
+}