omp-conductor 0.19.4 → 0.19.6

package/src/upgrade.ts

@@ -1,5 +1,6 @@
 import { existsSync, readFileSync } from "node:fs";
-import { join } from "node:path";
+import { homedir } from "node:os";
+import { dirname, join } from "node:path";
 import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
 import { inspectBriefLayout, type BriefLayout } from "./brief-upgrade.ts";
 import { pauseInstance, setPaused, statusSnapshot, type AdmissionAckRecord } from "./daemon.ts";
@@ -24,6 +25,7 @@ import {
   upgradeJournalPath,
   type UpgradeCheck,
   type UpgradeJournalEntry,
+  type InstallSnapshotManagement,
 } from "./upgrade-journal.ts";
 import {
   enqueueUpgradeReport,
@@ -129,6 +131,8 @@ export interface UpgradeDeps {
    */
   pauseState(project?: string): { source: string; reason?: string; since: number } | undefined;
   setPaused(value: boolean, project?: string): void;
+  /** Host-global recovery fence for host-wide install surfaces. */
+  setGlobalRecoveryPaused(value: boolean): void;
   restartDaemon(): Promise<void>;
   sleep(ms: number): Promise<void>;
   /**
@@ -205,6 +209,14 @@ export const DEFAULT_DEPS: UpgradeDeps = {
   pauseState: (project) => pauseInstance(project),
   setPaused: (v, project) =>
     setPaused(v, { source: "upgrade", reason: "upgrade, draining" }, project),
+  setGlobalRecoveryPaused: (v) => {
+    const current = pauseInstance();
+    if (v && current === undefined) {
+      setPaused(true, { source: "upgrade-recovery", reason: "upgrade recovery verification required" });
+    } else if (!v && current?.source === "upgrade-recovery") {
+      setPaused(false);
+    }
+  },
   restartDaemon: async () => {
     await restartDaemon({});
   },
@@ -269,7 +281,7 @@ function parseRegistry(raw: string): { version: string; gitHead: string } {
 interface ReleaseIdentityPlan {
   version: string;
   gitHead: string;
-  /** What `bun add -g` / `omp plugin install` are given for this identity. */
+  /** What the discovered CLI root / `omp plugin install` are given for this identity. */
   packageSpec: string;
   /** Named checks already run to earn this identity, journalled as evidence. */
   checks: UpgradeCheck[];
@@ -360,7 +372,12 @@ async function resolveIdentity(deps: UpgradeDeps, options: UpgradeOptions): Prom
   return { version, gitHead: sha, packageSpec: `github:${PACKAGE_SOURCE}#${sha}`, checks, bootstrap: true };
 }
 
-function ompPluginVersion(raw: string): string | undefined {
+interface OmpPluginEntry {
+  version: string;
+  path?: string;
+}
+
+function ompPluginInventory(raw: string): unknown[] {
   let parsed: unknown;
   try {
     parsed = JSON.parse(raw);
@@ -372,15 +389,51 @@ function ompPluginVersion(raw: string): string | undefined {
   }
   const npm = Reflect.get(parsed, "npm");
   if (!Array.isArray(npm)) throw new Error("omp plugin list returned no npm inventory");
-  const plugin = npm.find(
+  return npm;
+}
+
+function ompPluginEntry(raw: string): OmpPluginEntry | undefined {
+  const plugin = ompPluginInventory(raw).find(
     (entry) => entry !== null && typeof entry === "object" && Reflect.get(entry, "name") === PACKAGE,
   );
-  if (plugin === undefined) return undefined;
+  if (plugin === undefined || plugin === null || typeof plugin !== "object") return undefined;
   const version = Reflect.get(plugin, "version");
   if (typeof version !== "string" || version.length === 0) {
     throw new Error("installed omp-conductor plugin has no version");
   }
-  return version;
+  const path = Reflect.get(plugin, "path");
+  return {
+    version,
+    ...(typeof path === "string" && path.length > 0 ? { path } : {}),
+  };
+}
+
+function ompPluginVersion(raw: string): string | undefined {
+  return ompPluginEntry(raw)?.version;
+}
+
+function nodeModulesRoot(path: string): string | undefined {
+  const marker = "/node_modules/";
+  const at = path.lastIndexOf(marker);
+  return at <= 0 ? undefined : path.slice(0, at);
+}
+
+function defaultOmpPluginRoot(env: NodeJS.ProcessEnv): string {
+  const configured = env["OMP_CODING_AGENT_DIR"];
+  if (configured !== undefined && configured.length > 0) return join(configured, "plugins");
+  return join(env["HOME"] ?? homedir(), ".omp", "plugins");
+}
+
+function ompPluginInstallRoot(raw: string, env: NodeJS.ProcessEnv): string {
+  const inventory = ompPluginInventory(raw);
+  const ownPath = ompPluginEntry(raw)?.path;
+  const fallbackPath = inventory
+    .map((entry) =>
+      entry !== null && typeof entry === "object" ? Reflect.get(entry, "path") : undefined,
+    )
+    .find((path): path is string => typeof path === "string" && path.length > 0);
+  const root = nodeModulesRoot(ownPath ?? fallbackPath ?? "");
+  return root ?? defaultOmpPluginRoot(env);
 }
 
 function herdrPluginSource(raw: string): string | undefined {
@@ -393,12 +446,143 @@ function herdrPluginSource(raw: string): string | undefined {
   return source;
 }
 
-/** The three installed identities one host carries: the Bun-global CLI/daemon
- *  tree, the omp plugin, and the herdr recovery plugin's pin. */
+function defaultHerdrInstallRoot(env: NodeJS.ProcessEnv): string {
+  const config =
+    env["HERDR_CONFIG_DIR"] ??
+    join(env["XDG_CONFIG_HOME"] ?? join(env["HOME"] ?? homedir(), ".config"), "herdr");
+  return join(config, "plugins", "github");
+}
+
+function herdrPluginManagement(
+  raw: string,
+  env: NodeJS.ProcessEnv,
+): InstallSnapshotManagement["herdr"] {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw);
+  } catch {
+    throw new Error("herdr plugin list returned invalid JSON");
+  }
+  const result =
+    parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
+      ? Reflect.get(parsed, "result")
+      : undefined;
+  const plugins =
+    result !== null && typeof result === "object" && !Array.isArray(result)
+      ? Reflect.get(result, "plugins")
+      : undefined;
+  if (!Array.isArray(plugins)) throw new Error("herdr plugin list returned no plugin inventory");
+  const plugin = plugins.find(
+    (entry) => entry !== null && typeof entry === "object" && Reflect.get(entry, "plugin_id") === HERDR_PLUGIN,
+  );
+  const root =
+    plugin !== undefined && typeof plugin === "object"
+      ? Reflect.get(plugin, "plugin_root")
+      : undefined;
+  const source =
+    plugin !== undefined && typeof plugin === "object"
+      ? Reflect.get(plugin, "source")
+      : undefined;
+  const managedPath =
+    source !== null && typeof source === "object" && !Array.isArray(source)
+      ? Reflect.get(source, "managed_path")
+      : undefined;
+  const marker = "/plugins/github/";
+  const markerAt = typeof root === "string" ? root.indexOf(marker) : -1;
+  const installRoot =
+    typeof managedPath === "string" && managedPath.length > 0
+      ? dirname(managedPath)
+      : markerAt >= 0 && typeof root === "string"
+        ? root.slice(0, markerAt + marker.length - 1)
+        : defaultHerdrInstallRoot(env);
+  return {
+    ...(typeof root === "string" && root.length > 0 ? { root } : {}),
+    installRoot,
+  };
+}
+
+function cliInstallRoot(target: string): string {
+  const marker = `/node_modules/${PACKAGE}/`;
+  const at = target.lastIndexOf(marker);
+  if (at <= 0) {
+    throw new Error(
+      `installed omp-conductor CLI target ${target} is not inside a discoverable node_modules/${PACKAGE} root`,
+    );
+  }
+  return target.slice(0, at);
+}
+
+/** Discover the exact managed roots the transaction will mutate (#1019). */
+export async function discoverInstallManagement(
+  deps: SurfaceProbeDeps,
+): Promise<InstallSnapshotManagement> {
+  const session = resolveHerdrSession(deps.env);
+  const executable = (await mustRun(deps, "which", ["omp-conductor"])).stdout.trim();
+  if (executable.length === 0) throw new Error("which omp-conductor returned no executable");
+  const [resolved, omp, herdr] = await Promise.all([
+    mustRun(deps, "readlink", ["-f", executable]),
+    mustRun(deps, "omp", ["plugin", "list", "--json"]),
+    mustRun(deps, "herdr", ["--session", session, "plugin", "list", "--json"]),
+  ]);
+  const target = resolved.stdout.trim();
+  if (target.length === 0) throw new Error(`cannot resolve installed CLI ${executable}`);
+  const ompRoot = ompPluginInstallRoot(omp.stdout, deps.env);
+  return {
+    cli: { executable, target, root: cliInstallRoot(target) },
+    omp: { root: ompRoot, lock: join(ompRoot, "omp-plugins.lock.json") },
+    herdr: herdrPluginManagement(herdr.stdout, deps.env),
+  };
+}
+
+async function preflightInstall(
+  deps: UpgradeDeps,
+  management: InstallSnapshotManagement,
+  packageSpec: string,
+): Promise<void> {
+  const writable: { label: string; path: string }[] = [
+    { label: "CLI package root", path: management.cli.root },
+    { label: "CLI package manifest", path: join(management.cli.root, "package.json") },
+    { label: "CLI node_modules", path: join(management.cli.root, "node_modules") },
+    { label: "OMP plugin root", path: management.omp.root },
+    { label: "Herdr managed-plugin root", path: management.herdr.installRoot },
+  ];
+  if (existsSync(management.omp.lock)) {
+    writable.push({ label: "OMP plugin lock", path: management.omp.lock });
+  }
+  for (const target of writable) {
+    const checked = await deps.run("test", ["-w", target.path]);
+    if (checked.code !== 0) {
+      const detail = checked.stderr.trim() || checked.stdout.trim() || "missing or not writable";
+      throw new Error(
+        `upgrade preflight refused: ${target.label} ${target.path} is unusable (${detail}); ` +
+          "nothing has been installed or paused",
+      );
+    }
+  }
+  try {
+    await mustRun(deps, "bun", [
+      "add",
+      "--cwd",
+      management.cli.root,
+      "--exact",
+      "--dry-run",
+      packageSpec,
+    ]);
+  } catch (err) {
+    throw new Error(
+      `upgrade preflight refused: the CLI install command is unusable at ${management.cli.root} ` +
+        `(${err instanceof Error ? err.message : String(err)}); nothing has been installed or paused`,
+    );
+  }
+}
+
+/** The three installed identities one host carries: the discovered CLI package
+ * tree, the omp plugin, and the herdr recovery plugin's pin. */
 export interface InstalledSurfaces {
   cliVersion: string;
   ompVersion?: string;
   herdrSource?: string;
+  management?: InstallSnapshotManagement;
 }
 
 /** What reading the three surfaces needs — deliberately narrower than
@@ -417,6 +601,7 @@ export async function inspectSurfaces(
   deps: SurfaceProbeDeps,
   /** `readHerdr: false` skips the herdr session probe and plugin read
    *  entirely, so a host with no herdr on PATH can still be asked what its
+
    *  CLI and omp plugin are (#904). The upgrade transaction always reads all
    *  three — a release pins every surface. */
   opts: { readHerdr?: boolean } = {},
@@ -445,6 +630,56 @@ export async function inspectSurfaces(
   };
 }
 
+function packageVersionAt(deps: UpgradeDeps, manifest: string): string {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(deps.readFile(manifest));
+  } catch (err) {
+    throw new Error(
+      `cannot read restored package manifest ${manifest}: ${err instanceof Error ? err.message : String(err)}`,
+    );
+  }
+  const version =
+    parsed !== null && typeof parsed === "object" && !Array.isArray(parsed)
+      ? Reflect.get(parsed, "version")
+      : undefined;
+  if (typeof version !== "string" || version.length === 0) {
+    throw new Error(`restored package manifest ${manifest} has no version`);
+  }
+  return version;
+}
+
+/** Verify the snapshot's recorded destinations directly, before PATH-based probes. */
+function verifyRestoredManagement(deps: UpgradeDeps, previous: InstalledSurfaces): void {
+  const management = previous.management;
+  if (management === undefined) return;
+  const cliManifest = join(management.cli.root, "node_modules", PACKAGE, "package.json");
+  const cliVersion = packageVersionAt(deps, cliManifest);
+  if (cliVersion !== previous.cliVersion) {
+    throw new Error(
+      `restored CLI root ${management.cli.root} has ${cliVersion}, expected ${previous.cliVersion}`,
+    );
+  }
+  if (previous.ompVersion !== undefined) {
+    const ompManifest = join(management.omp.root, "node_modules", PACKAGE, "package.json");
+    const ompVersion = packageVersionAt(deps, ompManifest);
+    if (ompVersion !== previous.ompVersion) {
+      throw new Error(
+        `restored OMP plugin root ${management.omp.root} has ${ompVersion}, expected ${previous.ompVersion}`,
+      );
+    }
+  }
+  if (previous.herdrSource !== undefined) {
+    if (management.herdr.root === undefined) {
+      throw new Error("restored Herdr plugin had no recorded root to verify");
+    }
+    const manifest = join(management.herdr.root, "herdr-plugin.toml");
+    if (deps.readFile(manifest).trim().length === 0) {
+      throw new Error(`restored Herdr plugin manifest ${manifest} is empty`);
+    }
+  }
+}
+
 export function expectedHerdrSource(gitHead: string): string {
   return `github:${HERDR_SOURCE}@${gitHead}`;
 }
@@ -492,7 +727,7 @@ function previousHerdrInstall(source: string): readonly [string, readonly string
  * that represents them.
  *
  * Everything an upgrade or a draining restart replaces is host-wide: the
- * Bun-global CLI, the omp plugin, the Herdr plugin, and the one dispatch
+ * discovered CLI package tree, the omp plugin, the Herdr plugin, and the one
  * daemon. That daemon serves *every* configured project — the generated unit's
  * ExecStart carries no `--project` — so its restart lands on all of them at
  * once. The scope therefore names every project whose workers must drain and
@@ -1092,13 +1327,22 @@ export async function rollbackUpgrade(
       // Exact post-rollback identity verification decides whether absence is safe.
     }
   };
-
   if (installTouched) {
-    await restore("Bun-global omp-conductor CLI", "bun", [
-      "add",
-      "-g",
-      `${PACKAGE}@${previous.cliVersion}`,
-    ]);
+    if (previous.management === undefined) {
+      await restore("legacy Bun-global omp-conductor CLI", "bun", [
+        "add",
+        "-g",
+        `${PACKAGE}@${previous.cliVersion}`,
+      ]);
+    } else {
+      await restore(`omp-conductor CLI at ${previous.management.cli.root}`, "bun", [
+        "add",
+        "--cwd",
+        previous.management.cli.root,
+        "--exact",
+        `${PACKAGE}@${previous.cliVersion}`,
+      ]);
+    }
     if (previous.ompVersion === undefined) {
       await removeIfPresent("remove newly installed omp plugin", "omp", [
         "plugin",
@@ -1182,6 +1426,7 @@ export async function rollbackUpgrade(
 
   if (installTouched) {
     try {
+      verifyRestoredManagement(deps, previous);
       const restored = await inspectSurfaces(deps);
       if (
         restored.cliVersion !== previous.cliVersion ||
@@ -1262,6 +1507,19 @@ export async function upgradeConductor(
   // already passed (#908). Everything below reads the same pair either way.
   const identity = await resolveIdentity(deps, options);
   const release = { version: identity.version, gitHead: identity.gitHead };
+  const requestedProjects = (deps.env["OMP_CONDUCTOR_INSTALL_PROJECTS"] ?? "")
+    .split(",")
+    .filter((name) => name.length > 0);
+  const requestedHolder = deps.env["OMP_CONDUCTOR_INSTALL_HOLDER"];
+  const installHolder =
+    requestedHolder === "human" || requestedHolder === "orchestrator"
+      ? requestedHolder
+      : undefined;
+  const installProjects: readonly (string | undefined)[] =
+    requestedProjects.length === 0 ? scope.selectors : requestedProjects;
+  const installAttribution =
+    `Host-global install holder: ${installHolder ?? "unknown"}; affected projects: ` +
+    `${installProjects.map((selector) => selector ?? "(default)").join(", ")}.`;
   if (identity.bootstrap) {
     // The bootstrap's evidence lands in the journal before the first surface
     // moves, so an interrupted bootstrap leaves a record of what was verified
@@ -1287,15 +1545,24 @@ export async function upgradeConductor(
       version: release.version,
       gitHead: release.gitHead,
       unit: deps.env["OMP_CONDUCTOR_UNIT"],
+      selectors: scope.selectors.map((selector) => selector ?? null),
+      ...(requestedProjects.length === 0 ? {} : { installProjects: requestedProjects }),
+      ...(installHolder === undefined ? {} : { installHolder }),
     });
   }
   const initial = deps.layers(scope.pauseKey);
-  const surfaces = await inspectSurfaces(deps);
+  const observedSurfaces = await inspectSurfaces(deps);
   const briefs: ScopedBrief[] = scope.selectors.map((selector) => ({
     selector,
     ...deps.brief(selector),
   }));
-  const installNeeded = !surfacesCurrent(surfaces, release.version, release.gitHead);
+  const installNeeded = !surfacesCurrent(observedSurfaces, release.version, release.gitHead);
+  const management = installNeeded ? await discoverInstallManagement(deps) : undefined;
+  if (management !== undefined) await preflightInstall(deps, management, identity.packageSpec);
+  const surfaces: InstalledSurfaces = {
+    ...observedSurfaces,
+    ...(management === undefined ? {} : { management }),
+  };
   // The host-unit baseline, read while the OLD package is still the one
   // rendering (#905). A destination already differing from the old render was
   // edited outside conductor, so the reconcile after the install must not
@@ -1394,6 +1661,7 @@ export async function upgradeConductor(
         cliVersion: surfaces.cliVersion,
         ompVersion: surfaces.ompVersion,
         herdrSource: surfaces.herdrSource,
+        ...(surfaces.management === undefined ? {} : { management: surfaces.management }),
       },
       ...(preUpgradeBackup === undefined ? {} : { configBackup: preUpgradeBackup }),
       pauseKey: scope.pauseKey,
@@ -1413,7 +1681,14 @@ export async function upgradeConductor(
     deps.log("safety: pausing new issue claims");
     deps.setPaused(true, scope.pauseKey);
   }
-  journal({ kind: "phase", phase: "paused", surface: "dispatch", ok: true, version: release.version });
+  journal({
+    kind: "phase",
+    phase: "paused",
+    surface: "dispatch",
+    ok: true,
+    version: release.version,
+    pauseSince: deps.pauseState(scope.pauseKey)?.since,
+  });
   deps.log("drain: waiting for live omp worker sessions");
   try {
     await waitForDrain(deps, scope);
@@ -1436,9 +1711,18 @@ export async function upgradeConductor(
   const reloadStarted = Date.now();
   try {
     if (installNeeded) {
+      if (management === undefined) {
+        throw new Error("upgrade invariant failed: install needed without discovered surface management");
+      }
       installTouched = true;
-      deps.log(`install 1/3: Bun-global omp-conductor CLI → ${release.version}`);
-      await mustRun(deps, "bun", ["add", "-g", identity.packageSpec]);
+      deps.log(`install 1/3: omp-conductor CLI at ${management.cli.root} → ${release.version}`);
+      await mustRun(deps, "bun", [
+        "add",
+        "--cwd",
+        management.cli.root,
+        "--exact",
+        identity.packageSpec,
+      ]);
       journal({ kind: "phase", phase: "install", surface: "cli", ok: true, version: release.version, gitHead: release.gitHead });
       deps.log(`install 2/3: omp plugin omp-conductor → ${release.version}`);
       await mustRun(deps, "omp", ["plugin", "install", identity.packageSpec]);
@@ -1595,6 +1879,7 @@ export async function upgradeConductor(
     } catch {
       deps.setPaused(true, scope.pauseKey);
     }
+    deps.setGlobalRecoveryPaused(true);
     const failure = err instanceof Error ? err.message : String(err);
     journal({
       kind: "phase",
@@ -1623,10 +1908,11 @@ export async function upgradeConductor(
       journal({ kind: "outcome", phase: "rolled-back", ok: false, version: release.version, detail: failure });
       enqueueUpgradeReport(
         "tier2",
-        reportProject(scope.selectors),
+        reportProject(installProjects),
         `fleet upgrade to omp-conductor@${release.version} failed and was rolled back`,
         [
           `Reason: ${failure}`,
+          installAttribution,
           `Restored: cli=${surfaces.cliVersion}, omp=${surfaces.ompVersion ?? "missing"}, ` +
             `herdr=${surfaces.herdrSource ?? "missing"}`,
           ...(preUpgradeBackup === undefined ? [] : [`Pre-upgrade config snapshot: ${preUpgradeBackup}`]),
@@ -1639,10 +1925,11 @@ export async function upgradeConductor(
       journal({ kind: "outcome", phase: "rollback-failed", ok: false, version: release.version, detail: rollback });
       enqueueUpgradeReport(
         "tier2",
-        reportProject(scope.selectors),
+        reportProject(installProjects),
         `upgrade to omp-conductor@${release.version} failed and ITS ROLLBACK ALSO FAILED`,
         [
           `Failure: ${failure}`,
+          installAttribution,
           `Rollback failure: ${rollback}`,
           "The fleet may be at mixed versions; do not resume dispatch.",
           "Restore manually from the journal at " + upgradeJournalPath(stateDir()) + ".",
@@ -1748,6 +2035,9 @@ export async function rollbackFromJournal(
     cliVersion: snapshot.previous.cliVersion,
     ompVersion: snapshot.previous.ompVersion,
     herdrSource: snapshot.previous.herdrSource,
+    ...(snapshot.previous.management === undefined
+      ? {}
+      : { management: snapshot.previous.management }),
   };
   let configBefore: string | undefined;
   if (request.configBackup !== undefined && existsSync(request.configBackup)) {
@@ -1779,6 +2069,10 @@ export async function rollbackFromJournal(
     configBefore,
     request.configBackup,
   );
+  // The restored bytes remain host-wide unverified state until the returning
+  // daemon reads all three surfaces. A project-scoped request cannot leave its
+  // siblings admitting in that interval.
+  deps.setGlobalRecoveryPaused(true);
 
   const restoredVersion = previous.cliVersion;
   appendJournal(root, {
@@ -1791,10 +2085,17 @@ export async function rollbackFromJournal(
   });
   enqueueUpgradeReport(
     "tier2",
-    reportProject(request.selectors),
+    reportProject(request.installProjects ?? request.selectors),
     `fleet restored to omp-conductor@${restoredVersion} after the failed upgrade to ${request.version}`,
     [
       "The transient rollback unit restored every surface from the durable snapshot;",
+      `Host-global install holder: ${request.installHolder ?? "unknown"}; affected projects: ${
+        (request.installProjects ?? request.selectors).length === 0
+          ? "all configured projects"
+          : (request.installProjects ?? request.selectors)
+              .map((selector) => selector ?? "(default)")
+              .join(", ")
+      }.`,
       "dispatch stays paused until the first tick after these restarts verifies the old version.",
       `Pre-upgrade config snapshot: ${request.configBackup ?? "none recorded"}`,
     ],

package/src/verbs/actions.ts

@@ -1008,7 +1008,12 @@ export function githubVerbActions(
           return { code: ran.ok ? 0 : 1, stdout: ran.stdout, stderr: ran.stderr };
         },
         env(),
-        { kind: "install", version: execution.version },
+        {
+          kind: "install",
+          version: execution.version,
+          holder: execution.holder,
+          projects: execution.projects,
+        },
       );
       if (!launched.ok) return { ok: false, stderr: launched.stderr };
       return { ok: true, detail: launched.unit };