omp-conductor 0.17.0 → 0.18.0

package/src/setup-host.ts

@@ -15,10 +15,13 @@ import {
   DEFAULT_PORT,
   healthCheck,
   livingDaemon,
+  probeUnit,
   startDaemon,
   stopDaemon,
+  SYSTEMD_UNIT,
   type DaemonRecord,
   type StopResult,
+  type UnitOwnership,
 } from "./lifecycle.ts";
 import {
   legacyArmedMarkerPath,
@@ -48,7 +51,7 @@ export const SYSTEMD_UNIT_DIR = "/etc/systemd/system";
 export const RECOVER_SERVICE_NAME = "omp-conductor-recover.service";
 
 /** The playbook file shipped in the package's `systemd/` directory. */
-const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh";
+export const RECOVER_SCRIPT_FILE = "omp-conductor-recover.sh";
 
 /**
  * Where `setup host` installs the recovery playbook so the unit's ExecStart is
@@ -1512,8 +1515,125 @@ export function writeHostRuntime(plan: HostRuntimePlan): HostRuntimeWrite {
   return { wrote, warnings };
 }
 
+// ---------------------------------------------------------- rollback (#652) --
+//
+// The inverse of the writes above, for the setup apply's mutation inventory:
+// every path the apply can write is captured byte-for-byte (or as absence, or
+// as unreadable — never conflated) before the first mutation, and restored on
+// any later failure. `capturePathState` is the read half, `restorePathState`
+// the write half; the transaction that owns them lives in `setup-wizard.ts`.
+
+/** One path's pre-entry state, three-valued so absence is never confused with
+ *  unreadability:
+ *  - `absent` — nothing at the path; rollback removes what the apply created.
+ *  - `bytes` — readable on entry (a regular file with its mode, or a symlink
+ *    with its target); rollback writes those exact bytes/mode or re-links it.
+ *  - `unreadable` — something exists but cannot be read. The transaction must
+ *    fail closed before changing anything: collapsing this to absence and
+ *    later deleting the path would remove state the apply never saw. */
+export type CapturedPathState =
+  | { path: string; kind: "absent" }
+  | { path: string; kind: "bytes"; bytes: string; mode: number; symlink: boolean }
+  | { path: string; kind: "unreadable" };
+
+/** Reads {@link CapturedPathState} for one path. Symlinks are captured as
+ *  their target (readlink), never followed — the brief link is a symlink by
+ *  contract, and a dangling one is still state worth restoring. */
+export function capturePathState(path: string): CapturedPathState {
+  let st: Stats;
+  try {
+    st = lstatSync(path);
+  } catch {
+    return { path, kind: "absent" };
+  }
+  try {
+    if (st.isSymbolicLink()) {
+      return { path, kind: "bytes", bytes: readlinkSync(path, "utf8"), mode: st.mode & 0o7777, symlink: true };
+    }
+    return { path, kind: "bytes", bytes: readFileSync(path, "utf8"), mode: st.mode & 0o7777, symlink: false };
+  } catch {
+    return { path, kind: "unreadable" };
+  }
+}
+
+/** Options for {@link restorePathState}. */
+export interface RestorePathOptions {
+  /**
+   * Never remove or replace a regular file: only a symlink (or absence) is
+   * touched, so an operator's file that appeared mid-transaction survives a
+   * rollback that cannot clobber it — the same guard the brief-link write
+   * itself applies (#652). A regular file where the entry state said
+   * "absent"/"symlink" therefore reports a restoration failure: the prior
+   * state is not coherent and the operator must be told.
+   */
+  preserveRegularFile?: boolean;
+}
+
+/** Restores {@link CapturedPathState} captured before the first mutation:
+ *  writes the entry bytes back (with their mode), re-links a symlink's entry
+ *  target, or removes what the apply created when nothing was there. Returns
+ *  `undefined` on a complete restore, else the first failure's message — a
+ *  rollback that could not restore a path must be reported, never implied
+ *  complete. */
+export function restorePathState(state: CapturedPathState, opts: RestorePathOptions = {}): string | undefined {
+  const { path } = state;
+  try {
+    if (state.kind === "bytes" && state.symlink) {
+      // Re-link the entry target. Under `preserveRegularFile` a regular file
+      // that replaced the link mid-flight is left alone — and reported — so a
+      // rollback never clobbers an operator's file it did not create.
+      let current: Stats | undefined;
+      try {
+        current = lstatSync(path);
+      } catch {
+        current = undefined;
+      }
+      if (current !== undefined && !current.isSymbolicLink() && opts.preserveRegularFile === true) {
+        return `could not restore the symlink at ${path}: a regular file now occupies it`;
+      }
+      rmSync(path, { force: true });
+      mkdirSync(dirname(path), { recursive: true });
+      symlinkSync(state.bytes, path);
+      return undefined;
+    }
+    if (state.kind === "bytes") {
+      mkdirSync(dirname(path), { recursive: true });
+      writeFileSync(path, state.bytes);
+      // Mode is part of the pre-entry state (a staged unit 0644, the tick
+      // config 0600, the recovery playbook 0755): writeFileSync preserves an
+      // existing file's mode, so chmod explicitly restores the captured one.
+      chmodSync(path, state.mode);
+      return undefined;
+    }
+    // Absent at entry: remove what the apply created. A regular file that
+    // appeared where the entry was absent is the operator's own unless the
+    // caller says otherwise.
+    let current: Stats | undefined;
+    try {
+      current = lstatSync(path);
+    } catch {
+      return undefined; // already gone — a complete restore.
+    }
+    if (current.isSymbolicLink() || opts.preserveRegularFile !== true) {
+      rmSync(path, { force: true });
+      return undefined;
+    }
+    return `could not restore ${path}: a regular file appeared where nothing was at entry`;
+  } catch (err) {
+    return `could not restore ${path}: ${err instanceof Error ? err.message : String(err)}`;
+  }
+}
+
 export interface SetupSmokeResult {
   mode: "temporary" | "existing";
+  /**
+   * Whether the smoke itself proved the daemon answers `/healthz` on
+   * {@link daemon}.port. A record-less active unit has no port to probe, so
+   * its "existing" smoke is unproven: it cannot count as passed until the
+   * caller restarts the daemon through the lifecycle seam, whose
+   * `waitForOwnedDaemon` proves MainPID + `/healthz` (#651, review #2).
+   */
+  healthProven: boolean;
   status: StatusSnapshot;
   daemon: DaemonRecord;
 }
@@ -1522,6 +1642,15 @@ export interface SetupSmokeDeps {
   paused(project: string): boolean;
   runOnce(project: string): Promise<void>;
   living(): DaemonRecord | undefined;
+  /**
+   * systemd ownership of the `omp-conductor.service` supervisor unit, probed
+   * independently of the pidfile/runtime record. An active unit is a live
+   * supervised daemon even when its record is missing or unreadable (#618):
+   * it must never be run beside a `--once` tick, and `start()` must never be
+   * aimed at it. `unknown` is preserved — never collapsed into `inactive` —
+   * so a possibly-live supervised daemon is not run beside (#651, review #2).
+   */
+  unitOwnership(): UnitOwnership;
   health(port: number): Promise<{ ok: boolean; body?: string }>;
   start(project: string): Promise<DaemonRecord>;
   stop(): Promise<StopResult>;
@@ -1532,30 +1661,90 @@ const DEFAULT_SMOKE_DEPS: SetupSmokeDeps = {
   paused: isPaused,
   runOnce: async (project) => await runDaemon({ once: true, project }),
   living: livingDaemon,
+  unitOwnership: () => probeUnit(SYSTEMD_UNIT),
   health: healthCheck,
   start: async (project) => await startDaemon({ project }),
   stop: stopDaemon,
   status: statusSnapshot,
 };
 
+/** The daemon record a smoke attributes to a live but record-less supervisor.
+ *  The port is unknown (the record would have named it); the "existing" mode
+ *  caller restarts the daemon through the lifecycle seam and recomputes the
+ *  real /healthz line from that restart, so this satisfies the result shape
+ *  without pretending to know a port. `healthProven: false` marks that this
+ *  is a placeholder until that restart proves the unit's MainPID answers. */
+function unrecordedDaemon(mainPid: number): DaemonRecord {
+  return {
+    pid: 0,
+    port: 0,
+    startedAt: 0,
+    logFile: `<record missing — supervised daemon MainPID ${mainPid} via systemd>`,
+  };
+}
+
 export async function runSetupSmoke(
   project: string,
   deps: SetupSmokeDeps = DEFAULT_SMOKE_DEPS,
 ): Promise<SetupSmokeResult> {
   if (!deps.paused(project)) throw new Error("setup smoke requires paused dispatch");
-  await deps.runOnce(project);
+  // Detect a live daemon BEFORE the mutating `--once` tick (#618). A paused
+  // `--once` run is itself a dispatcher: it settles rows, projects labels,
+  // admits and salvages workers. Running one beside a live daemon orphans and
+  // salvages the runs the live process owns — exactly what the incident did to
+  // three workers before failing its restart check. An already-active daemon
+  // needs only its health proving (the caller restarts it through the lifecycle
+  // seam so the new config takes effect) — no `--once` process ever runs beside
+  // it, and no `systemctl start` is aimed at a unit that is already active.
   const existing = deps.living();
   if (existing !== undefined) {
     const health = await deps.health(existing.port);
     if (!health.ok) throw new Error(`existing daemon on :${existing.port} did not answer /healthz`);
-    return { mode: "existing", status: deps.status(project), daemon: existing };
+    return { mode: "existing", healthProven: true, status: deps.status(project), daemon: existing };
   }
 
+  // The pidfile record is absent, but that is not "no daemon": an active
+  // systemd-owned unit is a live supervised dispatcher even when its record
+  // went missing (a crash raced a re-record, or an unmanaged start never wrote
+  // one). Without this independent probe the smoke runs a second `--once`
+  // beside it, and `start()` targets the already-active unit — two dispatchers
+  // owning the same state, the #618 incident shape. Treat it as an existing
+  // daemon: no competing tick, no activation; the caller restarts the unit so
+  // the new config takes effect (#651, review #1).
+  const ownership = deps.unitOwnership();
+  if (ownership.kind === "unknown") {
+    // A probe failure is not the confirmed negative `inactive` claims to be:
+    // collapsing `unknown` here is how a unit-owned daemon gets a second one
+    // started beside it (#651, review #2). The mutating `--once` tick refuses
+    // instead.
+    throw new Error(
+      `cannot prove whether a daemon is running: systemd ownership probe failed (${ownership.reason}) — ` +
+        "refusing to run the setup smoke beside a possibly-live daemon",
+    );
+  }
+  if (ownership.kind === "active") {
+    // No record to read a port from, so no /healthz proof is possible here:
+    // the smoke reports `existing` honestly as *unproven*, and the caller
+    // must restart the unit through the lifecycle seam — whose
+    // `waitForOwnedDaemon` proves MainPID + `/healthz` — before the smoke
+    // counts as passed (#651, review #2).
+    return {
+      mode: "existing",
+      healthProven: false,
+      status: deps.status(project),
+      daemon: unrecordedDaemon(ownership.pid),
+    };
+  }
+
+  // Confirmed inactive (or failed, which owns no process): a paused `--once`
+  // smoke can run by itself, then a temporary daemon proves the new runtime,
+  // then it is stopped again.
+  await deps.runOnce(project);
   const daemon = await deps.start(project);
   try {
     const health = await deps.health(daemon.port);
     if (!health.ok) throw new Error(`temporary daemon on :${daemon.port} did not answer /healthz`);
-    return { mode: "temporary", status: deps.status(project), daemon };
+    return { mode: "temporary", healthProven: true, status: deps.status(project), daemon };
   } finally {
     await deps.stop();
   }

package/src/setup-install.ts

@@ -30,12 +30,17 @@ import {
   type UpgradeScope,
 } from "./upgrade.ts";
 import {
+  graphConflictMessage,
   graphRepos,
   formatGraphSetup,
+  legacyReindexFiles,
+  legacyReindexNote,
   mcpEntry,
+  planGraphSetup,
   reindexScriptPath,
+  reindexUnitName,
   resolvePrereqs,
-  REINDEX_UNIT,
+  stagingConflict,
   unitPaths,
   writeGraphSetup,
   type GraphPrereqs,
@@ -131,16 +136,18 @@ function linuxOnly(deps: InstallDeps): string | undefined {
 }
 
 /**
- * Whether two files hold identical bytes. The "keep" side of setup-host.ts's
- * `actionFor`: a unit systemd already reads that matches what would be written
- * needs no reinstall. Reads can fail (permissions, vanished mid-flight) — a
- * read error is a mismatch, never silently "matches", because the whole point
- * is to make the plan only claim a unit install is redundant when it can see
- * the installed copy.
+ * Whether a file holds exactly the rendered bytes. The "keep" side of
+ * setup-host.ts's `actionFor`: a unit systemd already reads that matches what
+ * would be written needs no reinstall. Reads can fail (permissions, vanished
+ * mid-flight) — a read error is a mismatch, never silently "matches", because
+ * the whole point is to make the plan only claim a unit install is redundant
+ * when it can see the installed copy. Comparing against the rendered content
+ * rather than the staged file keeps the comparison read-only: staging now
+ * happens only after consent (#720), so nothing may be written to compare.
  */
-function sameFile(a: string, b: string): boolean {
+function sameContent(path: string, content: string): boolean {
   try {
-    return readFileSync(a, "utf8") === readFileSync(b, "utf8");
+    return readFileSync(path, "utf8") === content;
   } catch {
     return false;
   }
@@ -450,9 +457,10 @@ export interface GraphInstallOptions extends InstallDeps {
  * Staging and enabling alone installs a service that fails on every run: the
  * generated script runs under `set -euo pipefail` and `cd "<graphProject>"` as
  * its first act per repo, so a missing clone is a `cd` failure at 03:00 rather
- * than a graph. That is why `writeGraphSetup` already refused to call the old
- * install a finished job. So: prerequisites, then clones, then install, then
- * seed and verify — one preview, one confirm.
+ * than a graph. That is why staging was never allowed to call the install a
+ * finished job. So: prerequisites, then a conflict check, then clones, then
+ * install, then seed and verify — one preview, one confirm, and nothing
+ * staged before the confirm (#720).
  */
 export async function runGraphInstall(
   project: ProjectConfig,
@@ -494,7 +502,38 @@ export async function runGraphInstall(
     return { kind: "refused", reason: missing };
   }
 
-  const staged = writeGraphSetup(project, options.unitDir ?? SYSTEMD_UNIT_DIR);
+  const unitDir = options.unitDir ?? SYSTEMD_UNIT_DIR;
+  // The plan is computed eagerly — `planGraphSetup` renders every byte, so the
+  // prompt can print exactly what would be written and the unitsCurrent gate
+  // below can decide a re-run installs nothing. But rendering is read-only: no
+  // file is written here. Staging happens only after consent (#720), so the
+  // prompt's "stages once you confirm" is true of the staged tree when it is
+  // printed, and a declined run leaves every staged file byte-identical.
+  const plan = planGraphSetup(project, unitDir);
+  // Never overwrite another project's staged artefact: the stems derive from
+  // the project name, and two names can fold to one stem ("My Project" vs
+  // "my_project"), so an existing file's generated-for marker is checked
+  // before the consent prompt — a collision is refused before anything is
+  // asked, let alone written (#720).
+  const conflict = stagingConflict(project, [
+    plan.script.path,
+    plan.service.path,
+    plan.timer.path,
+    ...Object.values(unitPaths(project, unitDir)),
+  ]);
+  if (conflict !== undefined) {
+    const message = graphConflictMessage(conflict);
+    ui.notify(message, "error");
+    return { kind: "refused", reason: message };
+  }
+  // A host that ran the pre-#720 project-less version keeps its old files:
+  // their timer refreshes whichever project generated it last, forever. Name
+  // the remediation before the consent prompt — deleting files outside this
+  // command's own names is exactly how the overwrite happened in the first
+  // place, so it is host action, not install action.
+  const legacy = legacyReindexFiles();
+  if (legacy.length > 0) ui.notify(legacyReindexNote(legacy), "warning");
+
   const absent = repos.filter((r) => !existsSync(r.graphProject));
 
   // 2. Missing clones, as the operator. A root-owned index-only clone under the
@@ -507,17 +546,18 @@ export async function runGraphInstall(
   }));
 
   const blocked = linuxOnly(options);
-  const { service, timer } = unitPaths(options.unitDir ?? SYSTEMD_UNIT_DIR);
-  const from = unitPaths(stateDir());
-  // The units systemd already reads, compared to what staging just wrote — the
+  const stem = reindexUnitName(project);
+  const { service, timer } = unitPaths(project, unitDir);
+  // The units systemd already reads, compared to the rendered content — the
   // same distinction setup-host.ts draws between `installedAction` and
   // `service.action` (read the installed file, compare to the fresh content,
   // "keep" vs "update"). "Installed" is not enough: a green timer serving a
   // stale unit is the older bug, so when they differ the install runs. Only
   // when both units already match does the plan omit them — then just the
-  // outstanding clone and seed remain.
+  // outstanding clone and seed remain. Nothing is written to compare: staging
+  // is post-consent now, so the comparison is against the rendered bytes.
   const unitsCurrent =
-    blocked === undefined && sameFile(service, from.service) && sameFile(timer, from.timer);
+    blocked === undefined && sameContent(service, plan.service.content) && sameContent(timer, plan.timer.content);
   // 3. Install and enable, privileged. 4. Seed, in the SAME batch: the contract is
   //    one preview and one confirm, and a second confirm here also invented a
   //    third outcome — a declined seed — that neither the caller nor the
@@ -525,32 +565,52 @@ export async function runGraphInstall(
   const seed: PrivilegedStep[] =
     options.noSeed === true
       ? []
-      : [{ title: `seed the indexes (runs ${REINDEX_UNIT}.service once, minutes per repo)`, argv: ["systemctl", "start", `${REINDEX_UNIT}.service`] }];
+      : [{ title: `seed the indexes (runs ${stem}.service once, minutes per repo)`, argv: ["systemctl", "start", `${stem}.service`] }];
   const install: PrivilegedStep[] =
     blocked === undefined
       ? [
           ...(unitsCurrent
             ? []
             : [
-                { title: "install the reindex unit and timer", argv: ["install", "-m", "0644", from.service, from.timer, join(options.unitDir ?? SYSTEMD_UNIT_DIR, "")] },
+                { title: "install the reindex unit and timer", argv: ["install", "-m", "0644", plan.service.path, plan.timer.path, join(unitDir, "")] },
                 { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
-                { title: `enable ${REINDEX_UNIT}.timer`, argv: ["systemctl", "enable", "--now", `${REINDEX_UNIT}.timer`] },
+                { title: `enable ${stem}.timer`, argv: ["systemctl", "enable", "--now", `${stem}.timer`] },
               ]),
           ...seed,
         ]
       : [];
 
   if (blocked !== undefined) {
+    // A non-Linux host cannot run systemd, so staging IS the deliverable: the
+    // rendered files are written for the operator to copy to the box that will
+    // run them, and only the systemctl half is refused. There is no consent
+    // prompt on this path — nothing privileged to approve — so the write lands
+    // here, not in the (never reached) post-consent hook (#720, as #510).
+    const staged = writeGraphSetup(project, unitDir);
     ui.notify(`${blocked} ${service} and ${timer}`, "warning");
     return { kind: "staged", wrote: staged.written, reason: `${blocked} ${service}` };
   }
 
+  // What the post-consent staging actually wrote, surfaced by the outcomes
+  // that report it. Empty on a decline — the hook never ran.
+  let wrote: readonly string[] = [];
+  const beforeRun = async (): Promise<void> => {
+    // Consent has been given; this project's files may now land. Staging here —
+    // post-confirm, pre-step, under the same confirm that authorised the batch
+    // — is what makes the prompt's plan true of the staged tree when it is
+    // printed: a declined prompt never reaches this hook, so nothing is staged
+    // and a re-run still plans an install (#720). `writeGraphSetup`'s ownership
+    // backstop throws here with nothing written if a collision raced the
+    // pre-consent check.
+    wrote = writeGraphSetup(project, unitDir).written;
+  };
+
   const outcome = await runPrivileged([...clones, ...install], ui, {
     ...(options.privileged === undefined ? {} : { deps: options.privileged }),
     title: unitsCurrent ? "Clone the code-graph checkouts and seed them?" : "Clone, install and enable the code-graph timer?",
     answerKey: `install-code-graph.${project.name}`,
     preamble: [
-      `Staged: ${staged.written.join(", ")}.`,
+      `Stages ${[plan.script.path, plan.service.path, plan.timer.path].join(", ")} once you confirm — nothing is written before that.`,
       ...(clones.length === 0
         ? ["Every indexed clone already exists."]
         : [
@@ -561,8 +621,9 @@ export async function runGraphInstall(
       ...(unitsCurrent ? ["The reindex units are already installed and current — nothing to install."] : []),
       `Indexer: ${prereqs.indexer ?? "on PATH"}.`,
     ],
+    beforeRun,
   });
-  if (outcome.kind === "declined") return { kind: "declined", wrote: staged.written };
+  if (outcome.kind === "declined") return { kind: "declined", wrote };
   if (outcome.kind === "failed") {
     return { kind: "failed", reason: `${outcome.step.title} exited ${outcome.exitCode}` };
   }
@@ -571,10 +632,10 @@ export async function runGraphInstall(
   // answer: an unseeded graph is not usable until the timer first fires.
   if (options.noSeed === true) {
     ui.notify(
-      `Timer enabled; skipped the seeding run. The graph is NOT usable until ${REINDEX_UNIT}.timer first fires.`,
+      `Timer enabled; skipped the seeding run. The graph is NOT usable until ${stem}.timer first fires.`,
       "warning",
     );
-    return { kind: "installed", wrote: staged.written };
+    return { kind: "installed", wrote };
   }
 
   const health = await (options.probe ?? probeCodeGraph)(project);
@@ -583,13 +644,13 @@ export async function runGraphInstall(
     // Staged but not trusted. Reporting success here is how an operator learns
     // months later that no worker ever read an index.
     ui.notify(
-      [`Installed, but ${unhealthy.length} repo(s) did not verify: ${unhealthy.join(", ")}.`, "", staged.next].join("\n"),
+      [`Installed, but ${unhealthy.length} repo(s) did not verify: ${unhealthy.join(", ")}.`, "", plan.next].join("\n"),
       "error",
     );
     return { kind: "failed", reason: `unverified: ${unhealthy.join(", ")}` };
   }
   ui.notify(`Code graph installed and verified for ${repos.map((r) => r.name).join(", ")}.`, "info");
-  return { kind: "installed", wrote: staged.written };
+  return { kind: "installed", wrote };
 }
 
 /** The two prerequisites that make an enabled timer meaningful, or `undefined`. */
@@ -618,7 +679,7 @@ export function graphInstallable(project: ProjectConfig): GraphRepo[] {
   return graphRepos(project);
 }
 
-/** Where the reindex script lands, for the wizard's tail to name. */
-export function reindexScriptLocation(): string {
-  return reindexScriptPath();
+/** Where this project's reindex script lands, for the wizard's tail to name. */
+export function reindexScriptLocation(project: ProjectConfig): string {
+  return reindexScriptPath(project);
 }