omp-conductor 0.19.4 → 0.19.6

package/src/doctor.ts

@@ -85,6 +85,7 @@ import {
 import {
   herdrConductorPluginConfigDir,
   planHostRuntime,
+  RECOVER_SERVICE_NAME,
   STAGED_SERVICE_NAME,
   SYSTEMD_UNIT_DIR,
   tickCwdForProject,
@@ -229,6 +230,8 @@ export interface DoctorDeps {
   hasSystemd?: () => boolean;
   /** The installed unit text at an absolute path, or undefined when absent. */
   readUnit?: (path: string) => string | undefined;
+  /** Whether one installed systemd unit is currently failed; undefined when unreadable. */
+  unitFailed?: (name: string) => boolean | undefined;
   /** stat one path; undefined when it does not exist. */
   stat?: (path: string) => Stats | undefined;
   /** uid of a username (`id -u`), or undefined when it cannot be resolved. */
@@ -397,6 +400,18 @@ function defaultReadUnit(path: string): string | undefined {
   }
 }
 
+export function systemdStateFailed(state: string): boolean | undefined {
+  const normalized = state.trim();
+  if (normalized === "") return undefined;
+  return normalized === "failed";
+}
+
+function defaultUnitFailed(name: string): boolean | undefined {
+  const ran = spawnSync("systemctl", ["is-failed", name], { encoding: "utf8" });
+  if (ran.error !== undefined) return undefined;
+  return systemdStateFailed(ran.stdout ?? "");
+}
+
 function defaultStat(path: string): Stats | undefined {
   try {
     return statSync(path);
@@ -660,7 +675,7 @@ function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Findin
     return warnFinding(
       "db-backup",
       "conductor.db has no snapshot — losing it loses the audit trail with no recovery path",
-      "take a conductor.db snapshot (the daemon's cadence files one; restore-db needs one to restore from)",
+      "run `omp-conductor snapshot-db` (the daemon cadence uses the same SQLite snapshot primitive)",
     );
   }
   const newestMtime = snapshots
@@ -670,7 +685,7 @@ function dbBackupProbe(probes: Probes, cfg: ConductorConfig | undefined): Findin
     return warnFinding(
       "db-backup",
       `conductor.db is newer than every snapshot (${snapshots.length} file(s)) — the newest writes were never snapshot`,
-      "take a fresh conductor.db snapshot so restore-db has the current state",
+      "run `omp-conductor snapshot-db` so restore-db has the current state",
     );
   }
   return passFinding("db-backup", `conductor.db snapshot is fresh (${snapshots.length} file(s))`);
@@ -934,6 +949,22 @@ function recoveryProbe(probes: Probes): Finding {
   );
 }
 
+function recoveryStateProbe(probes: Probes): Finding {
+  const id = "systemd-recovery-state";
+  if (!probes.hasSystemd()) return passFinding(id, "no systemd directory on this host — nothing to check");
+  if (probes.readUnit(join(SYSTEMD_UNIT_DIR, RECOVER_SERVICE_NAME)) === undefined) {
+    return passFinding(id, "recovery unit is not installed — systemd-recovery owns that finding");
+  }
+  const failed = probes.unitFailed(RECOVER_SERVICE_NAME);
+  if (failed === false) return passFinding(id, `${RECOVER_SERVICE_NAME} is not failed`);
+  const fix =
+    `inspect \`systemctl status ${RECOVER_SERVICE_NAME}\` and \`journalctl -u ${RECOVER_SERVICE_NAME}\`; ` +
+    `fix the root cause, then run \`systemctl reset-failed ${RECOVER_SERVICE_NAME}\` and rerun doctor`;
+  return failed === true
+    ? warnFinding(id, `${RECOVER_SERVICE_NAME} is in failed state — its last recovery did not complete`, fix)
+    : warnFinding(id, `${RECOVER_SERVICE_NAME} state could not be read`, fix);
+}
+
 function installedUnitUser(probes: Probes): string | undefined {
   const unit = probes.readUnit(join(SYSTEMD_UNIT_DIR, STAGED_SERVICE_NAME));
   if (unit === undefined) return undefined;
@@ -1713,7 +1744,7 @@ function parseEnvKey(text: string, key: string): string | undefined {
 
 /**
  * Install-surface parity (#904). omp-conductor installs onto three surfaces —
- * the Bun-global CLI/daemon tree, the omp plugin, and the herdr recovery
+ * the discovered CLI package tree, the omp plugin, and the herdr recovery
  * plugin — and only a live `upgrade` invocation ever compared them. On a
  * fleet whose installs are manual they diverge silently: measured on this
  * host on 2026-08-22, the omp plugin sat on the withdrawn 0.18.1 release for
@@ -2002,6 +2033,7 @@ export async function runDoctor(projectName: string | undefined, opts: DoctorDep
   // the first one stands in for the rendering seam.
   findings.push(unitProbe(probes, projects[0], cfg));
   findings.push(recoveryProbe(probes));
+  findings.push(recoveryStateProbe(probes));
   findings.push(ownershipProbe(probes));
   // #541 seam checks, host-global: the live herdr config and the plugin's
   // config.env are single files on the host, not per-project facts.
@@ -2106,6 +2138,7 @@ export function defaultProbes(): Probes {
     repoLabels: defaultRepoLabels,
     hasSystemd: () => existsSync(SYSTEMD_UNIT_DIR),
     readUnit: defaultReadUnit,
+    unitFailed: defaultUnitFailed,
     stat: defaultStat,
     uidOf: defaultUidOf,
     dbIntegrity: defaultDbIntegrity,

package/src/fleet.ts

@@ -223,6 +223,46 @@ export function armState(projectName?: string): { path: string; armed: boolean }
   return { path, armed: resolveArmState(path, named).armed };
 }
 
+export type WatchMonitoring =
+  | { monitored: true }
+  | { monitored: false; reason: string };
+
+/** Whether a durable watch can be observed and wake an orchestrator tick (#1024). */
+export function watchMonitoringState(
+  layers: Pick<FleetLayers, "daemon" | "ticks">,
+): WatchMonitoring {
+  if (!layers.daemon.running) return { monitored: false, reason: "daemon not running" };
+  if (layers.ticks === "armed" || layers.ticks === "ungated") return { monitored: true };
+  if (layers.ticks === "disarmed") {
+    return {
+      monitored: false,
+      reason: "ticks disarmed — conditions may still be checked, but no tick can be woken",
+    };
+  }
+  return { monitored: false, reason: `ticks ${layers.ticks}` };
+}
+
+/** Cheap watch liveness read: no Herdr, Telegram, healthz, or systemd probes. */
+export function watchMonitoringForProject(projectName: string): WatchMonitoring {
+  const daemon = livingDaemon();
+  if (daemon === undefined) return { monitored: false, reason: "daemon not running" };
+  if (daemon.project !== undefined && daemon.project !== projectName) {
+    return { monitored: false, reason: `daemon serves ${daemon.project}, not ${projectName}` };
+  }
+  const tick = resolveTickConfig(projectName);
+  const ticks: TicksLayer =
+    tick.kind === "absent"
+      ? "no-heartbeat-config"
+      : tick.kind === "invalid"
+        ? "invalid-heartbeat-config"
+        : tick.config.armedFile === undefined
+          ? "ungated"
+          : armState(projectName).armed
+            ? "armed"
+            : "disarmed";
+  return watchMonitoringState({ daemon: { running: true }, ticks });
+}
+
 /**
  * Clears the arm gate for this project — and `wasArmed` is the gate the
  * heartbeat reads, not merely one file's presence.
@@ -2174,7 +2214,7 @@ export async function collectFleetStatus(projectName?: string): Promise<FleetSta
     telegram,
     codeGraph,
     brief: briefStatusLine(project),
-    decisions: decisionStatusLine(project.name),
+    decisions: decisionStatusLine(project.name, layers),
     failureClasses: failureClassBlock(project.name),
     workerPhases,
     intake: intakeStatusLine(project.name),
@@ -2307,7 +2347,10 @@ function failureClassBlock(projectName: string): string | undefined {
  * watches behind GitHub's checks is not a fleet waiting on its operator, and
  * lumping them in made `decisions 3 open` read as three unanswered questions.
  */
-export function decisionStatusLine(projectName: string): string | undefined {
+export function decisionStatusLine(
+  projectName: string,
+  layers: Pick<FleetLayers, "daemon" | "ticks">,
+): string | undefined {
   const path = dbPath();
   if (!existsSync(path)) return undefined;
   let store: Store | undefined;
@@ -2325,7 +2368,12 @@ export function decisionStatusLine(projectName: string): string | undefined {
       const hours = Math.max(0, Math.round((Date.now() - oldest.askedAt) / 3_600_000));
       line = `decisions ${questions.length} open (oldest ${hours}h)`;
     }
-    if (watches.length > 0) line += ` · watches ${watches.length}`;
+    if (watches.length > 0) {
+      const monitoring = watchMonitoringState(layers);
+      line += monitoring.monitored
+        ? ` · watches ${watches.length} monitored`
+        : ` · watches ${watches.length} durable, unmonitored (${monitoring.reason})`;
+    }
     return line;
   } catch {
     return undefined;

package/src/orchestrator-tick.ts

@@ -49,7 +49,13 @@ import { spawnSync } from "node:child_process";
 import { existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { dirname, isAbsolute, join, resolve } from "node:path";
 import { availabilityPrompt, interruptDisposition } from "./availability.ts";
-import { findProject, loadConfig, resolveReleaseGrants, stateDir } from "./config.ts";
+import {
+  findProject,
+  loadConfig,
+  resolveReleaseGrants,
+  resolveSharedInstallAuthority,
+  stateDir,
+} from "./config.ts";
 import {
   bridgeTokenBound,
   hasBotToken,
@@ -4393,9 +4399,14 @@ export default function orchestratorTickExtension(
       let grants: ResolvedGrants = DENIED_RELEASE_GRANTS;
       let external = true;
       try {
-        const project = findProject(loadConfig(), configuredProject);
+        const config = loadConfig();
+        const project = findProject(config, configuredProject);
+        const installAuthority = resolveSharedInstallAuthority(config.projects);
         projectName = project.name;
-        grants = resolveReleaseGrants(project);
+        grants = {
+          ...resolveReleaseGrants(project),
+          install: installAuthority.holder ?? "human",
+        };
         external = project.escalation.orchestrator === "external";
       } catch (err) {
         // A missing/unreadable config cannot open a release gate. Log only when

package/src/setup-host.ts

@@ -780,11 +780,11 @@ export function renderHerdrUnit(runtime: ServiceRuntime): string {
  * recovery.
  */
 export function renderRecoverUnit(runtime: ServiceRuntime, projectName: string | undefined): string {
-  // `RECOVER_PROJECT` is how the playbook addresses the tier-2 escalation it
-  // enqueues. It is a host-global unit installed once for the whole box, so a
-  // static per-project value is only unambiguous on a single-project host — a
-  // multi-project host (or a no-project, host-global install) leaves it unset
-  // and reports without a `--project` rather than guessing one (#510/#530).
+  // `RECOVER_PROJECT` scopes the recovery action and optional re-arm. It is
+  // host-global and installed once, so a static action scope is legitimate
+  // only on a single-project host. With several projects it stays unset; the
+  // playbook independently derives one deterministic outbox owner from the
+  // live config and names every affected project in the report (#1027).
   return [
     "[Unit]",
     "Description=omp-conductor fleet recovery (OnFailure handler)",
@@ -1566,11 +1566,11 @@ export function planHostRuntime(
   const installedPath = join(unitDir, STAGED_SERVICE_NAME);
   const installedHerdr = join(unitDir, DEFAULT_HERDR_UNIT);
   const recoverUnitPath = join(stateDir(), RECOVER_SERVICE_NAME);
-  // The recovery unit is host-global: one shared unit, installed once. A
-  // static RECOVER_PROJECT is only legitimate when there is exactly one
-  // project to be unambiguous about — a multi-project host (or a no-project
-  // install) leaves it unset so the escalation reports without attributing a
-  // sibling's crash to one project (#510/#530).
+  // The recovery unit is host-global: one shared unit, installed once.
+  // `RECOVER_PROJECT` scopes recovery action/re-arm only and is therefore
+  // omitted on a multi-project host. Report ownership is a separate runtime
+  // decision: the playbook reads the configured project set, chooses its first
+  // project as outbox owner, and lists every project as affected (#1027).
   const recoverProject = project === undefined || multiProject ? undefined : project.name;
   const recoverUnitContent = renderRecoverUnit(runtime, recoverProject);
   const recoverUnit: PlannedWrite<string> = {

package/src/setup-wizard.ts

@@ -617,7 +617,7 @@ const RELEASE_SHAPE_QUESTIONS: { readonly [K in (typeof RELEASE_SHAPES)[number]]
   // package floor's "nobody patches the running conductor" is this grant's
   // deny default.
   install:
-    "install — replace this host's installed conductor: the Bun-global CLI, omp plugin " +
+    "install — replace this host's installed conductor: the discovered CLI package, omp plugin " +
     "and Herdr plugin pinned to one published release, executed detached from the fleet",
 };
 

package/src/status-render.ts

@@ -564,8 +564,19 @@ export function formatFleetStatus(
         })()
       : `dispatch  ${layers.dispatch}`;
 
+  const upgradeRecoveryBlock =
+    s.upgradeRecovery === undefined
+      ? undefined
+      : [
+          `upgrade   recovery required for failed omp-conductor@${s.upgradeRecovery.failedVersion} (${s.upgradeRecovery.phase})`,
+          `  journal  ${s.upgradeRecovery.journalPath}`,
+          `  owed     ${s.upgradeRecovery.verificationOwed}`,
+          "  release  automatic after successful verification; operator acknowledgement: `omp-conductor resume`",
+        ].join("\n");
+
   return [
     dispatchLine,
+    ...(upgradeRecoveryBlock === undefined ? [] : [upgradeRecoveryBlock]),
     tickLine,
     ...(nextTickLine === undefined ? [] : [nextTickLine]),
     paneLine,
@@ -1007,6 +1018,16 @@ function formatProjectBody(
   lines.push(...formatSalvagedRuns(s.salvagedRuns));
   lines.push(...formatQuarantinedRuns(s.quarantinedRuns));
   lines.push(...formatOpenReports(s.openReports));
+  const withdrawals = s.handoffWithdrawals ?? [];
+  if (withdrawals.length > 0) {
+    lines.push("handoff withdrawals");
+    for (const withdrawal of withdrawals) {
+      lines.push(
+        `  ${withdrawal.id}  ${withdrawal.target}  ${new Date(withdrawal.at).toISOString()}  ` +
+          `by ${withdrawal.actor} — ${withdrawal.reason}; ${withdrawal.summary}`,
+      );
+    }
+  }
   lines.push(...formatDigestBacklog(s.digestBacklog));
   // The mediated-verb ledger (#972). Absent from this renderer since the block
   // was written (#133) — it was wired into `daemon.ts`'s copy, which was