omp-conductor 0.15.10 → 0.15.12

package/src/verbs/actions.ts

@@ -15,9 +15,13 @@
  */
 
 import { credentialedEnv, openRunPr, pushRunBranch, repoSlugFor, scrubUserinfo } from "../gitops.ts";
+import { livingDaemon } from "../lifecycle.ts";
+import { stateDir } from "../config.ts";
 import { ensureMirror } from "../worktree.ts";
+import { launchTransientUnit, readUpgradeJournal } from "../upgrade-journal.ts";
+import { classifyUpgrade } from "../upgrade-verify.ts";
 import type { ProjectConfig, ReleaseShape, RepoTarget } from "../types.ts";
-import type { ActionOutcome, ReleaseExecution, VerbActions } from "./server.ts";
+import type { ActionOutcome, InstallExecution, ReleaseExecution, VerbActions } from "./server.ts";
 
 /**
  * What this daemon can actually cut.
@@ -779,5 +783,73 @@ export function githubVerbActions(
           }
         : failed(pushed.result, pushed.argv);
     },
+
+    // The fleet-installs-itself request (#486). Two gates, then a spawn:
+    //
+    // 1. `npm view` must actually expose the version with a full gitHead —
+    //    "install it" may only name what npm can serve. Refused with npm's
+    //    own detail so the orchestrator learns why, not that something was
+    //    wrong.
+    // 2. A live daemon must exist to verify the install on its first tick
+    //    after the restart. A stopped daemon can never come back to do that,
+    //    so the request is refused and the operator is pointed at the plain
+    //    `upgrade` command, whose own process is its witness.
+    install: async (execution: InstallExecution): Promise<ActionOutcome> => {
+      if (livingDaemon() === undefined) {
+        return {
+          ok: false,
+          stderr:
+            "the dispatch daemon is not running, so nobody can verify this install after the restart " +
+            "it performs. Run `omp-conductor upgrade` from a shell instead.",
+        };
+      }
+      // One install at a time: the journal's newest request is either closed or
+      // in flight, and a second request while one is in flight would race it
+      // for the same restarts. The refusal names the state so the caller knows
+      // to wait for the verifier's outcome instead of retrying blind.
+      const pending = classifyUpgrade(readUpgradeJournal(stateDir()));
+      if (pending.kind !== "none" && pending.kind !== "closed") {
+        return {
+          ok: false,
+          stderr:
+            `an install is already in flight (journal verdict: ${pending.kind}); ` +
+            "the fleet verifies or rolls it back before accepting another request.",
+        };
+      }
+      const looked = await run(
+        ["npm", "view", `omp-conductor@${execution.version}`, "version", "gitHead", "--json"],
+        { env: env() },
+      );
+      if (!looked.ok) {
+        const detail = looked.stderr.trim() || looked.stdout.trim() || "the npm view command failed";
+        return {
+          ok: false,
+          stderr:
+            `npm does not expose omp-conductor@${execution.version}: ${detail}. ` +
+            "Only a published release with a full gitHead can be installed; the request was not started.",
+        };
+      }
+      const metadata = objectFromJson(looked.stdout);
+      const published = metadata === undefined ? undefined : String(metadata["version"] ?? "");
+      const gitHead = metadata === undefined ? undefined : String(metadata["gitHead"] ?? "");
+      if (published !== execution.version || !/^[0-9a-f]{40}$/i.test(gitHead ?? "")) {
+        return {
+          ok: false,
+          stderr:
+            `npm does not expose omp-conductor@${execution.version} with a full gitHead ` +
+            `(${published === undefined ? "unreadable metadata" : `saw ${published}`}); the request was not started.`,
+        };
+      }
+      const launched = await launchTransientUnit(
+        async (argv) => {
+          const ran = await run([...argv] as string[], { env: env() });
+          return { code: ran.ok ? 0 : 1, stdout: ran.stdout, stderr: ran.stderr };
+        },
+        env(),
+        { kind: "install", version: execution.version },
+      );
+      if (!launched.ok) return { ok: false, stderr: launched.stderr };
+      return { ok: true, detail: launched.unit };
+    },
   };
 }

package/src/verbs/protocol.ts

@@ -288,15 +288,40 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
       `release authority is the orchestrator's, and this is a ${role} session. ` +
       "A worker session holds no release grant whatever the config says.",
   },
+  conductor_install: {
+    name: "conductor_install",
+    mutating: true,
+    allowedRoles: ["orchestrator"],
+    description:
+      "Request the fleet upgrade itself to a published omp-conductor release (#486). The install " +
+      "shape gates it like a release shape and stays with the human by default; with the grant, " +
+      "the daemon validates the version against npm and starts a detached transient unit that " +
+      "pauses, drains, installs the CLI/omp plugin/Herdr plugin and reloads — outside this session " +
+      "and the daemon. The unit never declares its own success: the first tick after the restart " +
+      "verifies version, /healthz, ticks, pane and doctor, and reports through the durable outbox.",
+    args: {
+      version: {
+        type: "string",
+        required: true,
+        description:
+          "The exact published semver to install (e.g. 0.15.11). The request is refused when npm " +
+          "does not expose it with a full gitHead.",
+      },
+      rationale: RATIONALE_ARG,
+    },
+    roleRefusalText: (role) =>
+      `install authority is the orchestrator's, and this is a ${role} session. ` +
+      "A worker session holds no install grant whatever the config says.",
+  },
   conductor_pr_status: {
     name: "conductor_pr_status",
     mutating: false,
     allowedRoles: ["worker", "orchestrator"],
     description:
-      "Read one pull request's live state and current head: is it open and are its checks green. " +
-      "Optionally compare that head with one the caller already observed. This is the daemon's own " +
-      "merge-gate verdict, not a summary of it. A poll, not a watcher — call it again after waiting. " +
-      "A worker may omit prUrl and gets its own run's.",
+      "Read one pull request's live state and current head — open, merged, or closed — and the checks " +
+      "available for it (checks are unavailable for a merged or closed PR). Optionally compare that head " +
+      "with one the caller already observed. This is the daemon's own merge-gate verdict, not a summary of it. " +
+      "A poll, not a watcher — call it again after waiting. A worker may omit prUrl and gets its own run's.",
     args: {
       prUrl: {
         type: "string",

package/src/verbs/server.ts

@@ -139,6 +139,12 @@ export interface ReleaseExecution {
   environment?: string;
 }
 
+/** What one install request asks the privileged half to do, after policy said yes. */
+export interface InstallExecution {
+  /** The published semver to pin all three surfaces to, exactly as npm has it. */
+  version: string;
+}
+
 /**
  * The privileged half, as an interface so every check above it is testable
  * without a repository, a credential or a network.
@@ -163,6 +169,13 @@ export interface VerbActions {
   mergePr(prUrl: string, headSha: string): Promise<ActionOutcome>;
   setLabel(issue: number, label: string, action: "add" | "remove"): Promise<ActionOutcome>;
   release(execution: ReleaseExecution): Promise<ActionOutcome>;
+  /**
+   * Request the fleet upgrade itself to `version`: refuse when npm does not
+   * expose it, then start the detached transient unit that performs it and
+   * survives the restarts. The implementation owns the npm read and the
+   * systemd-run spawn (#486) — the server only gates and records.
+   */
+  install(execution: InstallExecution): Promise<ActionOutcome>;
 }
 
 export interface VerbDeps {
@@ -272,6 +285,19 @@ export interface ReleaseFacts {
   unsettledRuns: number;
   /** Of those, the ones with a pull request still open. */
   openPrs: number;
+  /**
+   * Of those, the ones with a live worker (claimed/running), as opposed to
+   * finished pushes. Absent when the caller could not distinguish (the plain
+   * refusal wording is used then).
+   */
+  liveWorkers?: number;
+  /**
+   * Of those, finished runs whose PR the tracker could not confirm merged yet
+   * — the settle sweep either has not run or could not read GitHub. Distinct
+   * from {@link liveWorkers} because only a live worker means the operator
+   * must wait on a process; these clear by themselves on the next sweep.
+   */
+  unconfirmedMerges?: number;
   /** Queue depth, or `undefined` when the tracker could not be read. */
   queueDepth: number | undefined;
   /** Current live-head workflow verdict for the released routed repository. */
@@ -286,7 +312,26 @@ export function releaseRequirementRefusal(
 ): string | undefined {
   for (const requirement of requires) {
     if (requirement === "runs-settled" && facts.unsettledRuns > 0) {
-      return `policy.release.requires includes runs-settled and ${facts.unsettledRuns} run(s) have not settled`;
+      // The same blocker reads differently depending on WHY the row is still
+      // active: a live worker is the operator's wait, an unconfirmed merge or
+      // an unrecorded sweep clears by itself on the next settle pass. The
+      // gate has the split; the message must hand it to the reader.
+      const live = facts.liveWorkers ?? 0;
+      const unconfirmed = facts.unconfirmedMerges ?? 0;
+      // openPrs counts verifiably-open PRs *and* unread ones (fail-closed for
+      // no-open-prs), so the verifiably-open sub-bucket is the difference.
+      const open = Math.max(0, (facts.openPrs ?? 0) - unconfirmed);
+      // Everything not named above: closed unmerged PRs and rows with no PR.
+      const remaining = facts.unsettledRuns - live - unconfirmed - open;
+      const parts: string[] = [];
+      if (live > 0) parts.push(`${live} worker(s) still running`);
+      if (open > 0) parts.push(`${open} pull request(s) still open`);
+      if (unconfirmed > 0) parts.push(`${unconfirmed} run(s) whose PR merge the settle sweep has not yet recorded`);
+      if (remaining > 0) parts.push(`${remaining} run(s) with no merged PR (closed or none yet)`);
+      return (
+        `policy.release.requires includes runs-settled and ${facts.unsettledRuns} run(s) have not settled` +
+        (parts.length === 0 ? "" : ` — ${parts.join("; ")}`)
+      );
     }
     if (requirement === "no-open-prs" && facts.openPrs > 0) {
       return `policy.release.requires includes no-open-prs and ${facts.openPrs} pull request(s) are still open`;
@@ -462,7 +507,7 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
       verb === "conductor_pr_update" ||
       verb === "conductor_label");
   const stopped =
-    spec.mutating && verb !== "conductor_release" && !orchestratorCompletion
+    spec.mutating && verb !== "conductor_release" && verb !== "conductor_install" && !orchestratorCompletion
       ? deps.fleetStop()
       : undefined;
   if (stopped !== undefined) {
@@ -473,7 +518,7 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
         "fleet-paused",
         `refused: ${stopped}. The pause refuses new claims and work-starting mutations. ` +
           "Completion verbs for runs admitted before the pause (conductor_pr_merge, conductor_pr_update_branch, " +
-          "conductor_pr_update, conductor_label) and conductor_release remain available.",
+          "conductor_pr_update, conductor_label), conductor_release and conductor_install remain available.",
       );
     }
   }
@@ -493,6 +538,8 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
       return labelVerb(deps, project, channel, args, refuse, allow);
     case "conductor_release":
       return releaseVerb(deps, project, channel, args, refuse, allow);
+    case "conductor_install":
+      return installVerb(deps, project, channel, args, refuse, allow);
     case "conductor_pr_status":
       return prStatusVerb(deps, project, channel, args, refuse, allow);
   }
@@ -641,9 +688,11 @@ async function prCreateVerb(
 export const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
 
 function runForPr(deps: VerbDeps, project: string, prUrl: string): RunRecord | undefined {
-  return deps.store
-    .recentRuns(project, deps.now() - PR_LOOKUP_WINDOW_MS)
-    .find((candidate) => candidate.prUrl === prUrl);
+  // Every attempt that recorded this PR, newest first — not `recentRuns`'
+  // newest-attempt-per-issue view. The row a PR belongs to is the newest
+  // attempt that owns it, and a requeued continuation that never inherited
+  // the URL must not hide its predecessor's ownership (#434).
+  return deps.store.runsForProjectPr(project, prUrl, deps.now() - PR_LOOKUP_WINDOW_MS)[0];
 }
 
 /**
@@ -1043,9 +1092,69 @@ async function releaseVerb(
   const health = wantsBase
     ? deps.store.baseHealth(project.name).find((row) => row.repo === repoName)
     : undefined;
+
+  // `runs-settled` (and `no-open-prs`) read the PR fact, not the row (#482):
+  // a `pushed-green` row whose PR has actually merged is settled work, and a
+  // merged PR is not an open one. The settle sweep records the same fact on
+  // its next tick; the gate must not wait an extra tick the operator reached
+  // this release by holding. Fail-closed stays fail-closed: a row whose PR
+  // state cannot be read is still unsettled, and an unknown PR is still
+  // treated as open.
+  const wantsRunFacts =
+    policy.release.requires.includes("runs-settled") ||
+    policy.release.requires.includes("no-open-prs");
+  let unsettledRuns = active.length;
+  let openPrs = active.filter((r) => r.prUrl !== undefined).length;
+  let liveWorkers: number | undefined;
+  let unconfirmedMerges: number | undefined;
+  if (wantsRunFacts) {
+    let unsettled = 0;
+    let open = 0;
+    let live = 0;
+    let unconfirmed = 0;
+    for (const run of active) {
+      if (run.state === "claimed" || run.state === "running") {
+        unsettled += 1;
+        live += 1;
+        // A live worker's PR is not yet confirmable; count it as open, the
+        // same fail-closed answer the old row-count gave it.
+        if (run.prUrl !== undefined) open += 1;
+        continue;
+      }
+      // pushed-pending / pushed-green: the worker is finished; the PR decides.
+      if (run.prUrl === undefined) {
+        unsettled += 1; // nothing to read; the sweep cannot settle it either
+        continue;
+      }
+      let pr: PrState | undefined;
+      try {
+        pr = await deps.tracker.prState(run.prUrl);
+      } catch {
+        pr = undefined;
+      }
+      if (pr === "merged") continue; // settled in fact; not an open PR
+      unsettled += 1;
+      if (pr === "open") open += 1;
+      else if (pr === undefined) {
+        // Cannot confirm the merge (or its absence): fail closed, and count
+        // the row as an unconfirmed merge. Also open — an unread PR may be
+        // open — so `no-open-prs` keeps its fail-closed refusal.
+        open += 1;
+        unconfirmed += 1;
+      }
+      // "closed": the row still owes a failure write from the sweep; blocks
+      // exactly as it did before, the message bucket covers it.
+    }
+    unsettledRuns = unsettled;
+    openPrs = open;
+    liveWorkers = live;
+    unconfirmedMerges = unconfirmed;
+  }
   const unmet = releaseRequirementRefusal(policy.release.requires, {
-    unsettledRuns: active.length,
-    openPrs: active.filter((r) => r.prUrl !== undefined).length,
+    unsettledRuns,
+    openPrs,
+    ...(liveWorkers === undefined ? {} : { liveWorkers }),
+    ...(unconfirmedMerges === undefined ? {} : { unconfirmedMerges }),
     queueDepth,
     ...(health === undefined ? {} : { baseCheck: health.verdict }),
     ...(health?.verdict === "red" && health.detail !== undefined
@@ -1152,14 +1261,66 @@ async function releaseVerb(
 }
 
 
+/**
+ * The fleet-installs-itself request (#486).
+ *
+ * Gated like a release act: the caller must be an orchestrator (spec), and the
+ * `install` shape must be granted to the orchestrator (release policy) — the
+ * package floor's "nobody patches the running conductor" is the deny default
+ * an operator deliberately moves, never something this verb works around.
+ *
+ * Everything past the gate is the privileged half's job: `actions.install`
+ * refussses a version npm does not expose and starts the detached transient
+ * unit. This verb records the request durably (the journal the post-restart
+ * tick verifies against) and tells the caller where the evidence will land.
+ */
+async function installVerb(
+  deps: VerbDeps,
+  project: ProjectConfig,
+  channel: VerbChannel,
+  args: Record<string, unknown>,
+  refuse: Refuse,
+  allow: Allow,
+): Promise<Verdict> {
+  const grants = resolveReleaseGrants(project);
+  const granted = releaseRefusal(grants, channel.role, "install");
+  if (granted !== undefined) {
+    return refuse("release-not-granted", `refused: ${granted.reason}`);
+  }
+
+  const version = args["version"];
+  if (typeof version !== "string" || version.trim().length === 0) {
+    return refuse(
+      "malformed-argument",
+      `refused: conductor_install needs a version, got ${JSON.stringify(version)}. ` +
+        "Name the exact published semver, e.g. 0.15.11.",
+    );
+  }
+
+  const outcome = await deps.actions.install({ version: version.trim() });
+  if (!outcome.ok) {
+    return refuse("action-failed", `refused: the install request failed:\n${outcome.stderr}`);
+  }
+  const unit = outcome.detail;
+  return allow(
+    `install requested: omp-conductor@${version} will be installed by a detached unit` +
+      `${unit === undefined ? "" : ` (${unit})`} that survives the restart. ` +
+      "The first tick after the fleet restarts verifies version, /healthz, ticks, pane and doctor, " +
+"and the outcome is reported through the durable outbox.",
+  );
+}
+
 /**
  * The one read verb, and the reason a worker reports a PR's real state rather
  * than its own recollection.
  *
- * Reuses `Tracker.verifyPr`, so the answer is literally the merge gate's own
- * verdict rather than a second opinion assembled here — including the
- * moved-head case, whose reason already names both shas. A worker that quotes
- * this has read a fact; a worker that says "should be green" has not.
+ * Reuses `Tracker.verifyPr` in read mode, so the answer is literally the merge
+ * gate's own verdict rather than a second opinion assembled here — including
+ * the moved-head case, whose reason already names both shas. A worker that
+ * quotes this has read a fact; a worker that says "should be green" has not.
+ * Read mode also reports a merged or closed PR as a fact (`ok: true`) instead
+ * of the merge gate's `expected OPEN` refusal, so an orchestrator can confirm
+ * release evidence on a just-merged PR (#389).
  */
 async function prStatusVerb(
   deps: VerbDeps,
@@ -1190,17 +1351,16 @@ async function prStatusVerb(
     prUrl = own.prUrl;
     issue = channel.issue;
   } else {
+    // A read resolves any syntactically valid PR URL — nothing to gate. A
+    // release-preparation PR the workflow opened (never a run row, and not
+    // necessarily in `routing.repos`) still has to be confirmable before the
+    // orchestrator cuts a release (#389). `runForPr` only decides which issue
+    // to record on the ledger; `verifyPr`'s `PR_URL` check rejects a URL that
+    // is not a pull request.
     if (typeof asked !== "string") {
       return refuse("malformed-argument", "refused: conductor_pr_status needs prUrl when it has no run to infer one from.");
     }
     const target = runForPr(deps, project.name, asked);
-    if (target === undefined && !prInProjectRouting(project, asked)) {
-      return refuse(
-        "pr-not-this-run",
-        `refused: ${asked} is not a pull request in ${project.name}'s routed repositories ` +
-          `(${Object.values(project.routing.repos).map(repoSlugFor).join(", ") || "none"}).`,
-      );
-    }
     prUrl = asked;
     issue = target?.issue;
   }
@@ -1209,7 +1369,10 @@ async function prStatusVerb(
   const expectedHead = typeof askedHead === "string" ? askedHead : undefined;
   let verification: PrVerification | undefined;
   try {
-    verification = await deps.tracker.verifyPr(prUrl, expectedHead);
+    // Read mode: a merged or closed PR is a fact to report, not the merge
+    // gate's `expected OPEN` refusal (#389). The mutating verbs never pass
+    // this, so their open-PR gate is untouched.
+    verification = await deps.tracker.verifyPr(prUrl, expectedHead, { read: true });
   } catch (err) {
     verification = undefined;
     deps.log(`verb pr_status could not read ${prUrl}: ${err instanceof Error ? err.message : String(err)}`);

package/src/worker.ts

@@ -11,8 +11,8 @@
  */
 
 import { createSession, disposeSession, SessionAdmissionClosedError, type AgentSessionLike } from "./omp.ts";
-import type { ReleaseBlockContext } from "./release-policy.ts";
-import type { Caps, ReleaseShape, ResolvedGrants, RunState } from "./types.ts";
+import type { GateShape, ReleaseBlockContext } from "./release-policy.ts";
+import type { Caps, ResolvedGrants, RunState } from "./types.ts";
 
 /** Structured evidence fields from the worker's final report. */
 const PR_URL_PATTERN = /^pr:\s*(https:\/\/github\.com\/\S+\/pull\/\d+)\s*$/im;
@@ -86,7 +86,7 @@ export interface WorkerOpts {
    */
   releaseGrants?: ResolvedGrants;
   /** Durable audit sink for rejected release/deploy calls. */
-  onReleaseBlocked?: (shape: ReleaseShape, context: ReleaseBlockContext) => void;
+  onReleaseBlocked?: (shape: GateShape, context: ReleaseBlockContext) => void;
   /** The child's pid, the instant it exists. See {@link VerbListener.bindPid}. */
   onSpawn?: (pid: number) => void;
   /** Control socket for that child, beside the run's own session directory. */