omp-conductor 0.18.0 → 0.18.2

package/src/upgrade-verify.ts

@@ -21,6 +21,7 @@
  * only runtime imports are leaves (journal, config, store).
  */
 
+import { readFileSync } from "node:fs";
 import { loadConfig, stateDir } from "./config.ts";
 import { DEFAULT_PORT } from "./lifecycle.ts";
 import { dbPath, openStore } from "./store.ts";
@@ -168,6 +169,11 @@ export interface PendingUpgradeRequest {
   selectors: readonly (string | undefined)[];
   initial?: UpgradeJournalEntry["initial"];
   configBackup?: string;
+  /**
+   * Epoch-ms when the install began (the journal snapshot's write time): the
+   * deadline the live orchestrator session must have restarted after (#832).
+   */
+  reloadAfterMs?: number;
 }
 
 /** The newest request plus everything journaled after it: the view one
@@ -189,6 +195,7 @@ export function pendingUpgradeRequest(
   const request = progression[0]!;
   if (request.ok === false) return undefined;
   const snapshot = progression.find((entry) => entry.kind === "snapshot");
+  const snapshotAt = snapshot === undefined ? undefined : Date.parse(snapshot.at);
   return {
     version: request.version ?? "unknown",
     gitHead: request.gitHead,
@@ -197,13 +204,164 @@ export function pendingUpgradeRequest(
     selectors: (snapshot?.selectors ?? []).map((selector) => selector ?? undefined),
     initial: snapshot?.initial,
     configBackup: snapshot?.configBackup,
+    ...(snapshotAt !== undefined && Number.isFinite(snapshotAt) ? { reloadAfterMs: snapshotAt } : {}),
   };
 }
 
 // ---------------------------------------------------------------------------
-// The post-restart verifier
+// The live-orchestrator session attestation (#832)
 // ---------------------------------------------------------------------------
 
+/**
+ * The live orchestrator surface one project's `/healthz` carries: the mode the
+ * fleet runs, and — when the daemon hosts the session itself — the extension
+ * version that session process actually LOADED. `undefined` means the daemon
+ * carried no attestation at all: it predates the contract, or the body is
+ * unreadable, and that is "not attested", never "the newest".
+ */
+export interface LiveOrchestratorFacts {
+  mode: "embedded" | "external" | "failed";
+  loaded?: string;
+}
+
+/**
+ * Parse one project's `orchestrator` block off a daemon `/healthz` body.
+ * Absent or malformed answers are `undefined` — the caller decides what
+ * missing evidence means (for a post-upgrade daemon: unprovable reload).
+ */
+export function orchestratorFactsFromHealth(
+  body: string | undefined,
+  project: string,
+): LiveOrchestratorFacts | undefined {
+  if (body === undefined) return undefined;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(body);
+  } catch {
+    return undefined;
+  }
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
+  const projects = Reflect.get(parsed, "projects");
+  if (!Array.isArray(projects)) return undefined;
+  const entry = projects.find(
+    (candidate) =>
+      candidate !== null &&
+      typeof candidate === "object" &&
+      !Array.isArray(candidate) &&
+      Reflect.get(candidate as object, "project") === project,
+  );
+  if (entry === undefined) return undefined;
+  const raw = Reflect.get(entry as object, "orchestrator");
+  if (raw === null || typeof raw !== "object" || Array.isArray(raw)) return undefined;
+  const record = raw as Record<string, unknown>;
+  const mode = record["mode"];
+  if (mode !== "embedded" && mode !== "external" && mode !== "failed") return undefined;
+  const loaded = record["loaded"];
+  return {
+    mode,
+    ...(typeof loaded === "string" ? { loaded } : {}),
+  };
+}
+
+/**
+ * Epoch-ms when one pid started, read off `/proc` — the live-process fact an
+ * external (pane-owned) orchestrator attestation is built from. `undefined`
+ * when the answer cannot be read; absence is never "it restarted".
+ */
+export function processStartTimeMs(pid: number): number | undefined {
+  try {
+    const stat = readFileSync(`/proc/${String(pid)}/stat`, "utf8");
+    // Field 22 (`starttime`) counts clock ticks since boot; the comm field can
+    // contain `)`, so split after the last one. Dropping the `pid (comm)`
+    // prefix shifts every index by three, so full-field 22 is `fields[19]` —
+    // `fields[21]` would be `rss`, an ever-present constant that never times a
+    // live restart. CLK_TCK is 100 on every Linux host herdr targets.
+    const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
+    const startTicks = Number(fields[19]);
+    const boot = readFileSync("/proc/stat", "utf8");
+    const bootSeconds = Number(/^btime\s+(\d+)/m.exec(boot)?.[1]);
+    if (!Number.isFinite(startTicks) || !Number.isFinite(bootSeconds)) return undefined;
+    return (bootSeconds + startTicks / 100) * 1000;
+  } catch {
+    return undefined;
+  }
+}
+
+/** The omp processes herdr claims for the fleet pane resolved to start times,
+ *  or why they could not be read. The probe seam keeps the upgrade hermetic:
+ *  the pure verdict below compares the starts against the install deadline. */
+export type PaneOmpProbe = { starts: readonly number[] } | { problem: string };
+
+/**
+ * What a live pane session must prove after an upgrade: every omp process the
+ * fleet pane currently runs must have STARTED at or after the instant the
+ * install began — a pane process still running from before the install loaded
+ * the pre-upgrade extension, and an empty pane is exactly the stale-or-missing
+ * reload the upgrade must not claim success over. `undefined` when the pane
+ * proves the reload.
+ */
+export function paneRestartProblem(starts: readonly number[], reloadAfterMs: number): string | undefined {
+  const stale = starts.find((startedAt) => startedAt < reloadAfterMs);
+  if (stale !== undefined) {
+    const at = new Date(stale).toISOString();
+    const deadline = new Date(reloadAfterMs).toISOString();
+    return `the fleet pane is still running the omp process it started at ${at} — before the install began at ${deadline}; its loaded conductor extension predates this upgrade`;
+  }
+  if (starts.length === 0) {
+    return "no live omp process is claimed by the fleet pane — the orchestrator session did not reload";
+  }
+  return undefined;
+}
+
+/**
+ * What one project's live session must prove after the reloads. `undefined`
+ * is the pass; a string is the problem that must fail the transaction.
+ *
+ *  - `embedded` — the daemon hosts the session: the live process ATTESTED the
+ *    extension version it loaded at start, so the loaded version must equal
+ *    the target release. A session that survived the upgrade answers its old
+ *    version and fails here.
+ *  - `failed` — the embedded session failed to start: no live session exists
+ *    to have reloaded, and the upgrade must not claim one.
+ *  - `external` — the fleet pane owns the session and the daemon cannot
+ *    attest its loaded code, so the reload is proven from the pane's live
+ *    process start times (via the pane probe): every omp process the pane
+ *    runs must have started at/after the install began. A herdr restart that
+ *    never happened (`herdrRestarted` false) leaves the pane untouched and
+ *    this leg unattestable — the transaction skips it rather than failing a
+ *    pane nothing was supposed to reload.
+ */
+export function sessionReloadProblem(o: {
+  facts: LiveOrchestratorFacts | undefined;
+  targetVersion: string;
+  reloadAfterMs: number;
+  herdrRestarted: boolean;
+  pane: PaneOmpProbe | undefined;
+}): string | undefined {
+  if (o.facts === undefined) {
+    return "the restarted daemon did not attest its orchestrator session — /healthz carried no orchestrator fact; the live extension's code cannot be proven";
+  }
+  if (o.facts.mode === "embedded") {
+    if (o.facts.loaded !== o.targetVersion) {
+      return `the live orchestrator session loaded omp-conductor ${o.facts.loaded ?? "(unattested)"}, not ${o.targetVersion} — it was not reloaded by this upgrade`;
+    }
+    return undefined;
+  }
+  if (o.facts.mode === "failed") {
+    return "the embedded orchestrator session failed to start after the restart — there is no live session on the new extension";
+  }
+  // external: the pane owns the session; the daemon cannot attest the pane's
+  // loaded code, so the transaction proves the reload from the pane's live
+  // process start times — a pane process that survived the herdr restart
+  // loaded the pre-upgrade extension.
+  if (!o.herdrRestarted) return undefined;
+  if (o.pane === undefined) {
+    return "the fleet pane could not be probed — its reload cannot be proven";
+  }
+  if ("problem" in o.pane) return `the fleet pane could not be probed: ${o.pane.problem}`;
+  return paneRestartProblem(o.pane.starts, o.reloadAfterMs) ?? undefined;
+}
+
 /**
  * Every injectable process/state seam the post-restart verifier touches.
  *
@@ -232,6 +390,18 @@ export interface UpgradeVerifyDeps {
   launchRollback(
     version: string,
   ): Promise<{ ok: true; unit: string } | { ok: false; stderr: string }>;
+  /**
+   * The herdr session the fleet pane lives in — the pane-side name for
+   * resolving the external orchestrator's live processes (#832).
+   */
+  herdrSession: string;
+  /**
+   * The live omp processes the fleet pane claims, resolved to their start
+   * times — the evidence an external orchestrator's reload is judged on
+   * (#832). `{ problem }` means the pane surface answered unreadably; the
+   * upgrade must not invent a pass from an unreadable pane.
+   */
+  probePaneOmp(session: string): Promise<PaneOmpProbe>;
   log(message: string): void;
   now?(): number;
 }
@@ -430,12 +600,49 @@ export async function runUpgradeChecks(
       checks.push({ name: "herdr", ok: layers.herdr === "active", detail: layers.herdr });
     }
   }
+  // One probe feeds both the health check and the session attestation below:
+  // the body the daemon answered is the same read the session check parses.
+  const health = layers.daemon.running
+    ? await liveHealth(deps, layers.daemon.port ?? DEFAULT_PORT)
+    : undefined;
   checks.push({
     name: "health",
-    ok: layers.daemon.running && (await liveHealth(deps, layers.daemon.port ?? DEFAULT_PORT)).ok,
+    ok: health?.ok === true,
     detail: layers.daemon.running ? undefined : "no daemon record",
   });
 
+  // The live-orchestrator session (#832): a successful install must leave the
+  // orchestrator process executing the newly installed extension, and package
+  // identities on disk say nothing about what the running session loaded. The
+  // restarted daemon attests its hosted session's loaded version on /healthz;
+  // an external (pane-owned) session is proven from the pane's live process
+  // start times. A reload that cannot be proven fails the transaction — the
+  // verifier rolls back rather than report a success the fleet is not running.
+  const healthBody = health?.body;
+  const herdrRestarted = request.initial?.herdr === "active";
+  const sessionRequired = herdrRestarted || (request.initial?.dispatch ?? "running") !== "stopped";
+  let sessionProblem = "";
+  if (sessionRequired && (layers.daemon.running || herdrRestarted)) {
+    const pane = await deps.probePaneOmp(deps.herdrSession).catch(
+      (err): PaneOmpProbe => ({
+        problem: err instanceof Error ? err.message : String(err),
+      }),
+    );
+    sessionProblem =
+      sessionReloadProblem({
+        facts: orchestratorFactsFromHealth(healthBody, deps.projectName),
+        targetVersion: request.version,
+        reloadAfterMs: request.reloadAfterMs ?? Date.now(),
+        herdrRestarted,
+        pane,
+      }) ?? "";
+  }
+  checks.push({
+    name: "session",
+    ok: sessionProblem === "",
+    ...(sessionProblem === "" ? {} : { detail: sessionProblem }),
+  });
+
   const doctor = await deps.doctor(deps.projectName);
   const failing = doctor.findings.filter((finding) => finding.status === "fail");
   const drift = failing.filter((finding) => PRIVILEGED_INSTALL_DRIFT.has(finding.id));

package/src/upgrade.ts

@@ -9,8 +9,9 @@ import {
   resolveHerdrSession,
   telegramStateDir,
 } from "./fleet.ts";
+import { herdrPaneOmpStarts } from "./fleet.ts";
 import type { DispatchLayer, FleetLayers } from "./status-render.ts";
-import { livingDaemon, restartDaemon } from "./lifecycle.ts";
+import { DEFAULT_PORT, healthCheck, livingDaemon, restartDaemon } from "./lifecycle.ts";
 import { configBackupDir, configPath, findProject, loadConfig, resolveCaps, stateDir, writeConfigRaw } from "./config.ts";
 import { renderBriefForProject } from "./setup.ts";
 import {
@@ -26,10 +27,15 @@ import {
 } from "./upgrade-journal.ts";
 import {
   enqueueUpgradeReport,
+  orchestratorFactsFromHealth,
+  paneRestartProblem,
   pendingUpgradeRequest,
+  processStartTimeMs,
   reportProject,
   requestProgression,
   runCommand,
+  sessionReloadProblem,
+  type PaneOmpProbe,
   type PendingUpgradeRequest,
   type UpgradeCommandResult,
 } from "./upgrade-verify.ts";
@@ -95,6 +101,25 @@ export interface UpgradeDeps {
   sleep(ms: number): Promise<void>;
   env: NodeJS.ProcessEnv;
   log(message: string): void;
+  /**
+   * Probe the running daemon's `/healthz`. The post-reload session
+   * verification reads the live orchestrator attestation off the answer
+   * (#832): the restarted daemon says which conductor version its orchestrator
+   * session process actually loaded.
+   */
+  health(port: number): Promise<{ ok: boolean; body?: string }>;
+  /**
+   * The herdr session the fleet pane lives in, for the external-orchestrator
+   * pane probe (#832).
+   */
+  herdrSession(): string;
+  /**
+   * Every omp process the fleet pane currently claims by herdr, resolved to
+   * its start time — the evidence an external (pane-owned) orchestrator
+   * reloaded after the install began (#832). `{ problem }` when the pane
+   * surface answers unreadably: absence of evidence is never a reload.
+   */
+  probePaneOmp(session: string): Promise<PaneOmpProbe>;
   /**
    * The host runtime this version's package renders, compared against what is
    * actually installed: the daemon and herdr units, the recovery unit and its
@@ -146,6 +171,9 @@ export const DEFAULT_DEPS: UpgradeDeps = {
   restartDaemon: async () => {
     await restartDaemon({});
   },
+  health: async (port) => healthCheck(port),
+  herdrSession: () => resolveHerdrSession(process.env),
+  probePaneOmp: (session) => herdrPaneOmpStarts(runCommand, session, processStartTimeMs),
   // The bare host-global plan — the same render the advisory's `setup host`
   // command would install: no per-project tail, the recovery unit encoding no
   // one project's name, and FLEET_CWDS derived from every configured project.
@@ -668,6 +696,133 @@ async function waitForRecovery(
   throw new Error(`upgrade verification failed: ${problem}`);
 }
 
+// ---------------------------------------------------------------------------
+// The live orchestrator session reload (#832)
+// ---------------------------------------------------------------------------
+
+/** One pass of the post-reload session verification (#832): read the restarted
+ * daemon's attestation (embedded orchestrator: the live session's loaded
+ * version; external: the pane probe's process starts) and answer whether
+ * every in-scope live session proves it reloaded the target release after
+ * the install began. `undefined` is the pass — every project the daemon
+ * serves must have attested, because the restart covered the whole daemon at
+ * once.
+ *
+ * The embedded leg is gated on the dispatch restart: a daemon that was
+ * already stopped when the transaction began never hosted a session the
+ * transaction could have reloaded, so its absence is not a failure. The
+ * external leg is gated on the herdr restart: a transaction that left the
+ * pane alone has no reason to demand proof the pane did anything.
+ */
+async function sessionVerifyProblem(
+  deps: UpgradeDeps,
+  targetVersion: string,
+  reloadAfterMs: number,
+  initial: FleetLayers,
+): Promise<string | undefined> {
+  const dispatchRestarted = initial.dispatch !== "stopped";
+  const herdrRestarted = initial.herdr === "active";
+  if (!dispatchRestarted && !herdrRestarted) return undefined;
+  if (!dispatchRestarted) {
+    // No daemon to attest: only the pane could have reloaded anything, so the
+    // pane probe decides alone, exactly as the external leg of the detached
+    // verifier does.
+    const pane = await deps.probePaneOmp(deps.herdrSession());
+    if ("problem" in pane) return `the fleet pane could not be probed: ${pane.problem}`;
+    return paneRestartProblem(pane.starts, reloadAfterMs) ?? undefined;
+  }
+  const port = deps.layers().daemon.port ?? DEFAULT_PORT;
+  const health = await deps.health(port);
+  if (!health.ok) return `the restarted daemon did not answer /healthz on :${port}`;
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(health.body ?? "");
+  } catch {
+    return `the restarted daemon's /healthz on :${port} is not JSON — the live session cannot be attested`;
+  }
+  const projects = projectsFromHealth(parsed);
+  if (projects === undefined) {
+    return `the restarted daemon's /healthz on :${port} carried no projects — the live session cannot be attested`;
+  }
+  const names: string[] = [];
+  for (const entry of projects) {
+    if (entry !== null && typeof entry === "object" && !Array.isArray(entry) && "project" in entry) {
+      if (typeof entry.project === "string") names.push(entry.project);
+    }
+  }
+  if (names.length === 0) {
+    return `the restarted daemon's /healthz on :${port} named no project — the live session cannot be attested`;
+  }
+  const pane = await deps.probePaneOmp(deps.herdrSession());
+  for (const project of names) {
+    const problem = sessionReloadProblem({
+      facts: orchestratorFactsFromHealth(health.body, project),
+      targetVersion,
+      reloadAfterMs,
+      herdrRestarted: herdrRestarted && modeFromHealth(parsed, project) === "external",
+      pane,
+    });
+    if (problem !== undefined) return problem;
+  }
+  return undefined;
+}
+
+/**
+ * The `projects` array of a daemon /healthz payload, narrowed without an
+ * unchecked cast: `undefined` when the payload is not an object with a
+ * `projects` array — the caller decides what an unreadable answer means.
+ */
+function projectsFromHealth(parsed: unknown): unknown[] | undefined {
+  if (parsed === null || typeof parsed !== "object" || Array.isArray(parsed)) return undefined;
+  if (!("projects" in parsed)) return undefined;
+  return Array.isArray(parsed.projects) ? parsed.projects : undefined;
+}
+
+/**
+ * One project's attested orchestrator mode off a parsed /healthz payload.
+ * `undefined` when the entry or its mode cannot be read.
+ */
+function modeFromHealth(parsed: unknown, project: string): string | undefined {
+  const projects = projectsFromHealth(parsed);
+  if (projects === undefined) return undefined;
+  for (const entry of projects) {
+    if (entry === null || typeof entry !== "object" || Array.isArray(entry)) continue;
+    if (!("project" in entry) || entry.project !== project) continue;
+    if (!("orchestrator" in entry)) return undefined;
+    const orchestrator = entry.orchestrator;
+    if (orchestrator === null || typeof orchestrator !== "object" || Array.isArray(orchestrator)) {
+      return undefined;
+    }
+    if (!("mode" in orchestrator) || typeof orchestrator.mode !== "string") return undefined;
+    return orchestrator.mode;
+  }
+  return undefined;
+}
+
+/**
+ * The verify leg after the reloads (#832): the upgrade only reports success
+ * once the live orchestrator process has proven it executes the newly
+ * installed extension. Polls the attestation within the same recovery window
+ * as the other post-reload checks — the restarted daemon needs seconds to
+ * boot its orchestrator child and answer — then throws into the transaction's
+ * rollback path, so a failed session reload is a durable, actionable failure
+ * and never a reported success.
+ */
+async function waitForSessionVerified(
+  deps: UpgradeDeps,
+  targetVersion: string,
+  reloadAfterMs: number,
+  initial: FleetLayers,
+): Promise<void> {
+  let problem = "the live orchestrator session did not attest a reload";
+  for (let attempt = 0; attempt < RECOVERY_ATTEMPTS; attempt += 1) {
+    problem = (await sessionVerifyProblem(deps, targetVersion, reloadAfterMs, initial)) ?? "";
+    if (problem === "") return;
+    await deps.sleep(RECOVERY_POLL_MS);
+  }
+  throw new Error(`upgrade verification failed: ${problem}`);
+}
+
 /** One project's brief, as the transaction found it. */
 interface ScopedBrief {
   /** The selector to address the project-aware deps and the CLI with. */
@@ -1047,6 +1202,9 @@ export async function upgradeConductor(
   let briefChanged = false;
   let herdrReloadStarted = false;
   let daemonReloadStarted = false;
+  // The instant the transaction began replacing surfaces: the deadline the
+  // live orchestrator session must prove it restarted after (#832).
+  const reloadStarted = Date.now();
   try {
     if (installNeeded) {
       installTouched = true;
@@ -1129,6 +1287,22 @@ export async function upgradeConductor(
     await deps.sleep(1_000);
     await waitForRecovery(deps, initial, scope);
 
+    // verify 3/3 (#832): package identities are on disk, but the orchestrator
+    // session process executes the module it LOADED at start — a session that
+    // survived the restarts keeps running the pre-upgrade extension no matter
+    // what the packages say. The transaction only reports success once the
+    // live session (daemon-attested for the embedded orchestrator, pane start
+    // times for the external one) proves it reloaded the installed release. A
+    // fleet that restarted nothing — dispatch stopped AND herdr inactive —
+    // has no live orchestrator to prove, so there is nothing to attest.
+    if (initial.dispatch !== "stopped" || initial.herdr === "active") {
+      deps.log("verify 3/3: the live orchestrator session loaded the installed extension");
+      await waitForSessionVerified(deps, release.version, reloadStarted, initial);
+      journal({ kind: "phase", phase: "verify", surface: "session", ok: true, version: release.version });
+    } else {
+      deps.log("verify 3/3: no live orchestrator session to attest (dispatch stopped, herdr inactive)");
+    }
+
     if (!initial.paused) deps.setPaused(false, scope.pauseKey);
     const restoredDispatch = deps.layers(scope.pauseKey).dispatch;
     if (restoredDispatch !== initial.dispatch) {

package/src/verbs/protocol.ts

@@ -371,6 +371,45 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
     roleRefusalText: (role) =>
       `review authority is the orchestrator's; this is a ${role} session, and a worker never returns its own PR for revision.`,
   },
+  conductor_pr_recover: {
+    name: "conductor_pr_recover",
+    mutating: true,
+    allowedRoles: ["orchestrator"],
+    description:
+      "Recover the missing pull request for a terminal run whose branch was preserved with no PR (#806): " +
+      "the daemon proves the run has no live worker, that no other PR already represents the issue, and that the " +
+      "routed branch still exists at the exact 40-hex head the run recorded, then opens the missing PR from that " +
+      "branch to the repo's configured default branch and records it on the run so review/merge settle normally. " +
+      "Automatically adopts and returns an existing PR at that exact head (idempotent). Refused for a live run, a " +
+      "closed or merged issue/PR, a replaced or force-pushed head, an unrelated open closer, a recorded PR with no " +
+      "readable state, or a repo the project no longer routes.",
+    args: {
+      issueUrl: {
+        type: "string",
+        required: true,
+        description:
+          "Full issue URL of the run to recover. Must be in this project's tracker repository; the run resolved is the newest attempt for it.",
+      },
+      branch: {
+        type: "string",
+        required: false,
+        description: "Optional assertion: must equal the run's recorded branch.",
+      },
+      base: {
+        type: "string",
+        required: false,
+        description: "Optional assertion: must equal the repo's configured defaultBranch.",
+      },
+      headSha: {
+        type: "string",
+        required: false,
+        description: "Optional assertion: must equal the run's recorded exact head.",
+      },
+      rationale: RATIONALE_ARG,
+    },
+    roleRefusalText: (role) =>
+      `pull-request recovery is the orchestrator's; this is a ${role} session, and a worker's PR is its own to push and leave.`,
+  },
   conductor_pr_status: {
     name: "conductor_pr_status",
     mutating: false,