omp-conductor 0.18.2 → 0.19.0

package/src/unblock.ts

@@ -38,6 +38,7 @@
 import { hasContinuationBudget, hasFailedAttemptBudget } from "./admission.ts";
 import { projectLabels } from "./label-projection.ts";
 import { LIVE_STATES } from "./store.ts";
+import { wakeDispatch } from "./wake.ts";
 import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts";
 
 /** What one `unblock` did, and the state it found around it. */
@@ -54,6 +55,10 @@ export interface UnblockOutcome {
    *  to "will the dispatcher hold this issue as issue-active?" that #178
    *  found this verb guessing at. */
   active: boolean;
+  /** What the resident-daemon wake did, when this unblock restored the queue
+   *  label and therefore made work claimable now (#878). Absent when nothing
+   *  was requeued: there would be nothing for a pass to find. */
+  woken?: string;
   /** Set when a live worker (a claimed/running run) is on the issue. The
    *  dispatcher holds a worker-backed run as issue-active unconditionally,
    *  while worker-free pushed occupancy can bypass it (#175) — so this is
@@ -242,9 +247,19 @@ export async function unblockIssue(
     if (stillPending > 0) labelSyncQueued = stillPending;
   }
 
+  // An unblock that actually restored the queue label has made work claimable
+  // now, so the resident daemon is poked instead of leaving the issue for the
+  // next scheduled pass (#878). Only on a real restore: a withheld or skipped
+  // requeue changed nothing to dispatch, and a wake following a no-op is a claim
+  // of progress that did not happen. Best effort by construction — the label
+  // mutation already committed, so an unreachable daemon is a reported line,
+  // never an error here.
+  const woken = requeued !== undefined ? await wakeDispatch(project.name) : undefined;
+
   return {
     cleared,
     ...counts,
+    ...(woken === undefined ? {} : { woken }),
     ...(live ? { live: true as const } : {}),
     ...(latest === undefined ? {} : { latest }),
     ...(held === undefined ? {} : { forced: true as const }),
@@ -418,5 +433,12 @@ export function formatUnblock(
     lines.push(`  queue      "${project.queueLabel}" left untouched (--no-requeue)`);
   }
 
+  // What happens next in time, not just in principle (#878): a restored label
+  // that woke the daemon claims on an immediate pass, and one that could not
+  // reach it waits for the scheduled pass. Both are honest; a report that says
+  // "eligible again" without saying which invites the reader to assume the
+  // faster one.
+  if (o.woken !== undefined) lines.push(`  dispatch   ${o.woken}`);
+
   return lines.join("\n");
 }

package/src/unit-reconcile.ts

@@ -0,0 +1,303 @@
+/**
+ * The bounded unit-template reconcile (#905).
+ *
+ * `upgrade` swaps the package that *renders* the host units, which silently
+ * invalidates the installed copies on every release that touches a template —
+ * and then hands the operator a chore: re-run `omp-conductor setup host`
+ * yourself. That advisory (#598) is worse than a chore, because `setup host`
+ * restarts `herdr-fleet.service` at the end and kills its own calling pane
+ * (#834). This module is the other half of that finding: the smallest thing
+ * that makes the installed units match the render, and nothing else.
+ *
+ * Two properties make it safe to run inside the upgrade transaction:
+ *
+ *   - **It is bounded to unit templates.** The staged copies of the daemon
+ *     unit, the recovery unit and its playbook, the herdr session unit and the
+ *     harness mount unit; then `install` for exactly the destinations whose
+ *     live bytes differ, then one `daemon-reload`. No account creation, no
+ *     ACLs, no `enable`, no `restart`, no herdr config merge, no tick config,
+ *     no brief link — every one of those stays `setup host`'s, and a change
+ *     that needs them is reported rather than attempted.
+ *   - **It never overwrites an operator's edit.** The caller names the
+ *     destinations that were *already* drifting before the new package landed;
+ *     those are hand edits rather than drift this upgrade created, and they are
+ *     skipped and reported. A host whose units already match is left untouched
+ *     byte for byte, because the render-and-compare in
+ *     {@link HostRuntimePlan.drift} is what decides.
+ *
+ * It lives in its own module, and runs as its own CLI verb, because the render
+ * is *code*: `upgradeConductor` replaces the package underneath itself, so
+ * every in-process render after the install phase is still the OLD version's
+ * templates. Only a freshly spawned process can render what the new release
+ * actually ships, which is why the transaction shells out to this verb instead
+ * of calling `writeHostRuntime` directly.
+ */
+
+import { writeFileSync } from "node:fs";
+import { dirname, join } from "node:path";
+
+import {
+  HARNESS_MOUNT_UNIT_NAME,
+  RECOVER_SCRIPT_INSTALL_PATH,
+  RECOVER_SERVICE_NAME,
+  STAGED_SERVICE_NAME,
+  type HostRuntimePlan,
+} from "./setup-host.ts";
+import { DEFAULT_HERDR_UNIT } from "./fleet.ts";
+import { runPrivileged, type PrivilegedDeps, type PrivilegedStep } from "./privileged.ts";
+import type { WizardUi } from "./wizard-ui.ts";
+
+/** The confirm key, so a non-interactive caller can answer it up front. */
+export const UNIT_RECONCILE_ANSWER_KEY = "reconcile-units";
+
+/** One staged file and the destination it belongs at. */
+export interface UnitDestination {
+  /** What this destination is, for the plan and the log. */
+  label: string;
+  /** The staged copy under the state directory — the render's own bytes. */
+  source: string;
+  /** The live path the system reads. */
+  installed: string;
+  /** `install -m` mode: units are data, the playbook is executed. */
+  mode: string;
+  /** The rendered content, so the staged copy can be written before install. */
+  content: string;
+}
+
+/**
+ * Every destination this reconcile is allowed to touch, in dependency order:
+ * the recovery playbook and unit before the daemon unit that names it in
+ * `OnFailure=`, so systemd never reloads a unit referencing one it cannot
+ * load.
+ *
+ * Deliberately absent, and not an oversight: the herdr pane-shell config and
+ * the herdr-conductor `config.env` (fleet-account files whose refresh implies
+ * a herdr restart), the worker identity, the tick config and the brief link.
+ * Those are `setup host`'s to own.
+ */
+export function unitDestinations(plan: HostRuntimePlan): UnitDestination[] {
+  const unitDir = dirname(plan.installedPath);
+  const destinations: UnitDestination[] = [
+    {
+      label: "recovery playbook",
+      source: plan.recoverScript.path,
+      installed: RECOVER_SCRIPT_INSTALL_PATH,
+      mode: "0755",
+      content: plan.recoverScript.content,
+    },
+    {
+      label: RECOVER_SERVICE_NAME,
+      source: plan.recoverUnit.path,
+      installed: join(unitDir, RECOVER_SERVICE_NAME),
+      mode: "0644",
+      content: plan.recoverUnit.content,
+    },
+    {
+      label: STAGED_SERVICE_NAME,
+      source: plan.service.path,
+      installed: plan.installedPath,
+      mode: "0644",
+      content: plan.service.content,
+    },
+  ];
+  if (plan.herdrUnit !== undefined) {
+    destinations.push({
+      label: DEFAULT_HERDR_UNIT,
+      source: plan.herdrUnit.path,
+      installed: join(unitDir, DEFAULT_HERDR_UNIT),
+      mode: "0644",
+      content: plan.herdrUnit.content,
+    });
+  }
+  return destinations;
+}
+
+export interface UnitReconcilePlan {
+  /** Destinations that will be refreshed, in execution order. */
+  refresh: UnitDestination[];
+  /**
+   * Host state this version retires, from {@link HostRuntimePlan.retire} (#895).
+   *
+   * It rides here rather than in a verb of its own because `upgrade` already
+   * spawns this reconcile as a child of the just-installed CLI — the only
+   * process that can render what the new release ships, and therefore the only
+   * one that knows what it stopped shipping. Retirement is NOT drift: a unit
+   * this version no longer renders cannot be compared against a render, so it
+   * is never protected by `--protect` and never counted as out of scope.
+   */
+  retire: readonly string[];
+  /** Drifted destinations left alone because the caller protected them. */
+  protectedDrift: string[];
+  /** Drifted destinations outside this reconcile's remit, named so the caller
+   *  can say precisely what still needs `setup host` — never a blanket
+   *  "re-run setup host". */
+  outOfScope: string[];
+  /** The privileged batch: one `install` per refreshed destination, then one
+   *  `daemon-reload`. Empty when nothing drifts. */
+  steps: PrivilegedStep[];
+}
+
+/**
+ * What reconciling this plan would do. Pure: it renders no file and runs no
+ * command, so a caller can print it, test it, or decide there is nothing to do.
+ *
+ * `protect` names destinations whose installed bytes already differed from the
+ * *previous* version's render — operator hand edits, which this never
+ * overwrites.
+ */
+export function planUnitReconcile(
+  plan: HostRuntimePlan,
+  protect: readonly string[] = [],
+  /** `false` retires only — the posture `upgrade` uses when it could not read
+   *  a drift baseline, so a refresh cannot tell an operator's edit from its
+   *  own, but a retirement still must not be skipped (#895). */
+  refreshDestinations = true,
+): UnitReconcilePlan {
+  const protectedSet = new Set(protect);
+  const drifted = new Set(plan.drift);
+  const known = unitDestinations(plan);
+  const refresh: UnitDestination[] = [];
+  const protectedDrift: string[] = [];
+  for (const destination of known) {
+    if (!refreshDestinations) continue;
+    if (!drifted.has(destination.installed)) continue;
+    if (protectedSet.has(destination.installed)) {
+      protectedDrift.push(destination.installed);
+      continue;
+    }
+    refresh.push(destination);
+  }
+  const mine = new Set(known.map((d) => d.installed));
+  const outOfScope = refreshDestinations ? plan.drift.filter((path) => !mine.has(path)) : [];
+  const retireSteps = plan.retire?.steps ?? [];
+  const steps: PrivilegedStep[] =
+    refresh.length === 0 && retireSteps.length === 0
+      ? []
+      : [
+          // Retirement first, and before any install: an obsolete unit is
+          // disabled and unmounted while the daemon this reconcile serves is
+          // still the one the caller drained for.
+          ...retireSteps,
+          ...refresh.map((destination) => ({
+            title: `install ${destination.label} (${destination.installed})`,
+            argv: ["install", "-m", destination.mode, destination.source, destination.installed],
+          })),
+          // One reload for the whole batch: systemd re-reads every unit file,
+          // so per-unit reloads would be the same work repeated. No `enable`
+          // and no `restart` — the units are already enabled, and restarting
+          // is the caller's decision, taken inside its own drain window.
+          { title: "reload systemd", argv: ["systemctl", "daemon-reload"] },
+        ];
+  const retire = plan.retire === undefined ? [] : [...plan.retire.units, ...plan.retire.staged];
+  return { refresh, retire, protectedDrift, outOfScope, steps };
+}
+
+export type UnitReconcileOutcome =
+  /** Nothing drifted and nothing to retire: the host already matches this
+   *  version. */
+  | { kind: "current"; protectedDrift: string[]; outOfScope: string[] }
+  /** The batch ran. `refreshed` are the destinations now matching the render,
+   *  and `retired` the host state this version removed (#895). */
+  | { kind: "reconciled"; refreshed: string[]; retired: string[]; protectedDrift: string[]; outOfScope: string[] }
+  /** The confirm was declined; nothing was written or installed. */
+  | { kind: "declined"; pending: string[] }
+  /** A step failed. `failed` names it; nothing after it ran. */
+  | { kind: "failed"; failed: string; detail: string; remaining: string[] };
+
+/**
+ * Stage the rendered unit files and install the ones whose live bytes differ.
+ *
+ * The staged writes happen first and unprivileged (they land under the state
+ * directory the fleet account owns), and only for destinations that will
+ * actually be installed — so a declined confirm leaves the state directory as
+ * it was, and a current host is not rewritten just to prove it is current.
+ */
+export async function reconcileUnits(
+  plan: HostRuntimePlan,
+  ui: WizardUi,
+  options: {
+    protect?: readonly string[];
+    /** `false` retires only, refreshing nothing (#895). */
+    refresh?: boolean;
+    /** Pre-answered confirm for a non-interactive caller (the upgrade
+     *  transaction, whose operator already consented to the whole upgrade). */
+    deps?: PrivilegedDeps;
+    /** Injected in tests; defaults to a real file write. */
+    write?: (path: string, content: string) => void;
+  } = {},
+): Promise<UnitReconcileOutcome> {
+  const reconcile = planUnitReconcile(plan, options.protect ?? [], options.refresh ?? true);
+  if (reconcile.steps.length === 0) {
+    return { kind: "current", protectedDrift: reconcile.protectedDrift, outOfScope: reconcile.outOfScope };
+  }
+  const write = options.write ?? ((path: string, content: string) => writeFileSync(path, content));
+  const outcome = await runPrivileged(reconcile.steps, ui, {
+    ...(options.deps === undefined ? {} : { deps: options.deps }),
+    title: "Install the re-rendered host units now?",
+    answerKey: UNIT_RECONCILE_ANSWER_KEY,
+    preamble: [
+      ...(reconcile.refresh.length === 0
+        ? []
+        : ["This release renders host unit files that differ from the installed copies."]),
+      ...(reconcile.retire.length === 0
+        ? []
+        : [`This release retires host state an earlier one installed: ${reconcile.retire.join(", ")}.`]),
+      "Units only: nothing is enabled, restarted, or re-provisioned here.",
+    ],
+    // Staged copies are written under the same consent as the install that
+    // reads them, and after it — never before a declined confirm.
+    beforeRun: async () => {
+      for (const destination of reconcile.refresh) write(destination.source, destination.content);
+    },
+  });
+  if (outcome.kind === "declined") {
+    return { kind: "declined", pending: [...reconcile.retire, ...reconcile.refresh.map((d) => d.installed)] };
+  }
+  if (outcome.kind === "failed") {
+    return {
+      kind: "failed",
+      failed: outcome.step.title,
+      detail: outcome.stderr.trim().split("\n")[0] ?? `exit ${outcome.exitCode}`,
+      remaining: outcome.remaining.map((step) => step.title),
+    };
+  }
+  return {
+    kind: "reconciled",
+    refreshed: reconcile.refresh.map((d) => d.installed),
+    retired: [...reconcile.retire],
+    protectedDrift: reconcile.protectedDrift,
+    outOfScope: reconcile.outOfScope,
+  };
+}
+
+/** One line per outcome, for the CLI and the upgrade journal. */
+export function formatUnitReconcile(outcome: UnitReconcileOutcome): string {
+  const tail = (protectedDrift: readonly string[], outOfScope: readonly string[]): string =>
+    [
+      protectedDrift.length === 0
+        ? ""
+        : ` left alone (modified outside conductor): ${protectedDrift.join(", ")}.`,
+      outOfScope.length === 0
+        ? ""
+        : ` still owed to \`omp-conductor setup host\`: ${outOfScope.join(", ")}.`,
+    ].join("");
+  switch (outcome.kind) {
+    case "current":
+      return `host units: already match this version's render.${tail(outcome.protectedDrift, outcome.outOfScope)}`;
+    case "reconciled":
+      return (
+        (outcome.refreshed.length === 0
+          ? "host units: nothing to refresh."
+          : `host units: reconciled ${outcome.refreshed.length} destination(s) — ${outcome.refreshed.join(", ")}.`) +
+        (outcome.retired.length === 0 ? "" : ` Retired: ${outcome.retired.join(", ")}.`) +
+        tail(outcome.protectedDrift, outcome.outOfScope)
+      );
+    case "declined":
+      return `host units: not reconciled (declined) — still drifted: ${outcome.pending.join(", ")}.`;
+    case "failed":
+      return (
+        `host units: reconcile failed at "${outcome.failed}" (${outcome.detail}); ` +
+        `${outcome.remaining.length} step(s) did not run.`
+      );
+  }
+}

package/src/upgrade-verify.ts

@@ -39,13 +39,20 @@ export interface UpgradeCommandResult {
   stderr: string;
 }
 
-export async function runCommand(command: string, args: readonly string[]): Promise<UpgradeCommandResult> {
+export async function runCommand(
+  command: string,
+  args: readonly string[],
+  /** Where to run it. Absent means this process's cwd, which is what every
+   *  step but the bootstrap's source checks wants (#908). */
+  cwd?: string,
+): Promise<UpgradeCommandResult> {
   try {
     const child = Bun.spawn([command, ...args], {
       stdin: "ignore",
       stdout: "pipe",
       stderr: "pipe",
       env: process.env,
+      ...(cwd === undefined ? {} : { cwd }),
     });
     const stdout = new Response(child.stdout).text();
     const stderr = new Response(child.stderr).text();