omp-conductor 0.15.11 → 0.15.13

package/src/failure-class.ts

@@ -222,6 +222,27 @@ export function dispatchInfra(
 // front of the thrown message, so a bare /^git / would never match a real row.
 const DISPATCH_GIT_ERROR = /^(?:Error: )?git .+ exited \d+/s;
 
+/** The prefix the draining/restarting process writes to a run it killed. */
+const ADMIN_RESTART_MARKER = "admin restart:";
+
+/**
+ * The actor a draining restart recorded on a run it killed, or `undefined`
+ * when the row carries no such attribution. Read from `lastError` for the
+ * same reason {@link dispatchInfra} is read from it: it is the one surface a
+ * run's own row already persists, and the restarting process writes it the
+ * way `recordOperatorStop` writes "operator stopped: …". Recognised as an
+ * administrative kill, never a charging failure, because nothing the worker
+ * did ended it — #512's gap was that a killed session left no attribution.
+ */
+export function adminRestartAttribution(lastError: string | undefined): string | undefined {
+  if (lastError === undefined) return undefined;
+  const marker = lastError.indexOf(ADMIN_RESTART_MARKER);
+  if (marker === -1) return undefined;
+  const firstLine = lastError.slice(marker + ADMIN_RESTART_MARKER.length).trim().split("\n")[0];
+  if (firstLine === undefined || firstLine.trim() === "") return undefined;
+  return firstLine.trim();
+}
+
 export function classifyRun(
   run: RunRecord,
   facts: ClassifyFacts,
@@ -300,6 +321,22 @@ export function classifyRun(
     }
   }
 
+  // A run the draining/restarting process attributed before it restarted the
+  // session host: killed under its own ceilings by an administrative action,
+  // not by anything the worker did. Costs no budget and requeues, naming the
+  // actor (#512). A killed session used to leave no attribution and fell
+  // through to `unknown`, charging a failure for an operator's restart.
+  if (run.state === "failed" || run.state === "killed") {
+    const actor = adminRestartAttribution(run.lastError);
+    if (actor !== undefined) {
+      return {
+        cls: "admin-kill",
+        recovery: "requeue",
+        evidence: `killed by an administrative restart (${actor}) — an operator's restart, not the run's work`,
+      };
+    }
+  }
+
   if (run.state === "blocked") {
     return {
       cls: "question",

package/src/fleet.ts

@@ -36,7 +36,7 @@ import { readApprovalSurface, readDaemonProfile } from "./approval-surface.ts";
 import { inspectBriefLayout } from "./brief-upgrade.ts";
 import { dbPath, openStore } from "./store.ts";
 import { renderBriefForProject } from "./setup.ts";
-import type { ProjectConfig, Store } from "./types.ts";
+import type { DaemonStop, ProjectConfig, Store } from "./types.ts";
 import { settlementFlagSummary } from "./diff-flags.ts";
 import { probeCodeGraph, type CodeGraphHealth } from "./graph-health.ts";
 import { formatDigestBacklog, formatOpenReports } from "./reports.ts";
@@ -52,6 +52,7 @@ import {
   statusSnapshot,
   type StatusSnapshot,
 } from "./daemon.ts";
+import { formatOrchestratorDown } from "./orchestrator-down.ts";
 import {
   healthCheck,
   isAlive,
@@ -1410,6 +1411,31 @@ function formatDaemonUnit(unit: UnitOwnership | undefined, recordPid: number): s
   }
 }
 
+/**
+ * The newest daemon stop/restart provenance as status lines (#378).
+ *
+ * Rendered under the daemon block whether the daemon is down or has since
+ * been restarted — the whole point is that the debrief survives the restart.
+ * An unattributed record says so explicitly: the external-signal fallback
+ * records what the receiving daemon knew (that it was unattributed, when,
+ * which projects it served with live counts) rather than guessing a caller.
+ */
+function formatLastStop(lastStop: DaemonStop | undefined): string[] {
+  if (lastStop === undefined) return [];
+  const caller = lastStop.unattributed
+    ? "unattributed — no mediated request (external signal)"
+    : `pid ${lastStop.callerPid ?? "?"}` +
+      (lastStop.callerUid === undefined ? "" : ` uid ${lastStop.callerUid}`) +
+      (lastStop.role === undefined ? "" : ` (${lastStop.role})`);
+  return [
+    `  last stop  ${new Date(lastStop.at).toISOString()}  ${lastStop.controlPath}`,
+    `  caller     ${caller}`,
+    `  scope      ${lastStop.scope}${lastStop.project === undefined ? "" : `: ${lastStop.project}`}`,
+    `  affects    ${lastStop.affected.map((a) => `${a.project} (${a.live} live)`).join(", ")}`,
+    `  reason     ${lastStop.reason}`,
+  ];
+}
+
 export function formatFleetStatus(
   s: StatusSnapshot,
   layers: FleetLayers,
@@ -1422,6 +1448,7 @@ export function formatFleetStatus(
   failureClasses: string | undefined = undefined,
   workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
   intake: string | undefined = undefined,
+  lastStop: DaemonStop | undefined = undefined,
 ): string {
   const tickLine =
     layers.ticksDetail === undefined
@@ -1505,8 +1532,9 @@ export function formatFleetStatus(
     ...(intake === undefined ? [] : [intake]),
     ...(graphBlock === undefined ? [] : [graphBlock]),
     daemonBlock,
+    ...formatLastStop(lastStop),
     "",
-    formatProjectBody(s, workerPhases),
+    formatProjectBody(s, workerPhases, now),
   ].join("\n");
 }
 
@@ -1540,12 +1568,18 @@ function formatDigestScheduleStatus(s: StatusSnapshot): string[] {
 function formatProjectBody(
   s: StatusSnapshot,
   workerPhases: ReadonlyMap<number, WorkerPausePhase> = new Map(),
+  now = Date.now(),
 ): string {
   const lines = [
     `project   ${s.project}${s.paused ? "  (PAUSED)" : ""}`,
     ...(s.pauseReason === undefined ? [] : [`paused    ${s.pauseReason}`]),
     `config    ${s.configPath}`,
     `state     ${s.stateDir}`,
+    // The orchestrator-down degrade row: first-class in the body, present only
+    // while the incident is open, so recovery drops it (#288).
+    ...(s.orchestratorDown === undefined
+      ? []
+      : formatOrchestratorDown(s.orchestratorDown, now)),
     ...formatAvailabilityStatus(s),
     ...formatDigestScheduleStatus(s),
     "",
@@ -1675,6 +1709,18 @@ export async function renderStatus(projectName?: string): Promise<string> {
   const cached = codeGraphFromHealthz(healthBody, project.name);
   const codeGraph = cached ?? (await probeCodeGraph(project));
   const workerPhases = workerPhasesFromHealthz(healthBody, project.name);
+  // The newest host-wide stop/restart provenance (#378). Read here — not in
+  // `statusSnapshot`, which is synchronous and belongs to the daemon module —
+  // and rendered identically from either project: the daemon_stops table is
+  // deliberately not partitioned by project, because the daemon serves every
+  // project and the uninvolved one must see who stopped it too.
+  const store = openStore(dbPath());
+  let lastStop: DaemonStop | undefined;
+  try {
+    lastStop = store.latestDaemonStop();
+  } finally {
+    store.close();
+  }
   return formatFleetStatus(
     { ...s, planUsage, github },
     layers,
@@ -1687,6 +1733,7 @@ export async function renderStatus(projectName?: string): Promise<string> {
     failureClassBlock(project.name),
     workerPhases,
     intakeStatusLine(project.name),
+    lastStop,
   );
 }
 

package/src/gitops.ts

@@ -17,6 +17,7 @@
  * branch cannot disagree.
  */
 
+import { existsSync } from "node:fs";
 import { join } from "node:path";
 
 import { parseChainSource, type ChainEntry } from "./chain-check.ts";
@@ -321,3 +322,159 @@ export async function readBaseChain(
     return { ok: false, stderr: err instanceof Error ? err.message : String(err) };
   }
 }
+
+// ------------------------------------------- the shared-host critical-base guard
+
+/**
+ * The stale-base admission verdict (#428): whether a preserved continuation
+ * branch may be reattached at all, once a project marks a base commit as
+ * critical. Fail closed — a branch that cannot be *proven* to contain every
+ * marker is refused.
+ */
+export type CriticalBaseVerdict =
+  /** No preserved branch for this issue in the mirror — a clean first attempt. */
+  | { state: "no-branch" }
+  /** The reattach source contains every configured marker. */
+  | { state: "fresh" }
+  /** The reattach source predates a marker and cannot be safely advanced to it,
+   *  so the dispatcher must not reattach it. `range` names the base commits
+   *  the branch has not absorbed, for the hold's recovery wording. */
+  | { state: "stale"; marker: string; range: string[] }
+  /** The marker could not be verified at all (unresolvable, fetch failed). */
+  | { state: "unknown"; error: string };
+
+/** The stubborn-session view of the probe, injectable for deterministic tests. */
+export type CriticalBaseProbe = (
+  repo: RepoTarget,
+  markers: readonly string[],
+  branch: string,
+) => Promise<CriticalBaseVerdict>;
+
+/** The bounded sample of base commits a held branch is missing. */
+const CRITICAL_BASE_RANGE_MAX = 8;
+
+/**
+ * Answer whether the preserved branch {@link addRunRepo} would reattach
+ * contains every configured critical-base marker (#428). A base safety fix
+ * protects only branches forked after it landed; a continuation forked before
+ * it still carries the dangerous code, and re-running the lifecycle suite it
+ * retains on a shared host is what SIGTERMed the production daemon.
+ *
+ * The reattach source is the mirror's `refs/heads/<branch>`. When it lacks a
+ * marker but the live remote branch now carries it and the reattach source is
+ * that live head's ancestor, the operator advanced the branch on GitHub (e.g.
+ * merged base into the PR branch): the fast-forward is folded into the mirror
+ * — safe and work-preserving, the same reconcile `pushRunBranch` performs —
+ * and the marker is accepted. Judged against the reattach source, never the
+ * live head alone, so a stale local copy cannot smuggle pre-fix code back in.
+ */
+export async function probeCriticalBase(
+  project: Pick<ProjectConfig, "mirrorRoot">,
+  repo: RepoTarget,
+  branch: string,
+  markers: readonly string[],
+  exec: Exec = spawnCaptured,
+): Promise<CriticalBaseVerdict> {
+  const mirror = mirrorPath(project, repo);
+  // A mirror is where preserved branches live; without one there is nothing to
+  // reattach. Judged "no branch" rather than "unknown" so a fleet that has not
+  // yet created a mirror is not deadlocked at admission.
+  if (!existsSync(mirror)) return { state: "no-branch" };
+  const branchRef = `refs/heads/${branch}`;
+  const env = credentialedEnv();
+
+  const present = await exec(
+    ["git", "--git-dir", mirror, "show-ref", "--verify", "--quiet", branchRef],
+    { env },
+  );
+  if (present.code !== 0) return { state: "no-branch" };
+
+  // The configured marker is a commit that landed on base, so base has to be
+  // resolvable to prove anything about the branch against it. Fetch today's
+  // base tip into the mirror's tracking refs first, so the judgement is against
+  // the current base rather than whatever a run last left behind.
+  const baseRef = `refs/remotes/origin/${repo.defaultBranch}`;
+  const fetched = await exec(
+    ["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${repo.defaultBranch}:${baseRef}`],
+    { env },
+  );
+  if (fetched.code !== 0) {
+    return {
+      state: "unknown",
+      error: scrubUserinfo(fetched.stderr.trim() || fetched.stdout.trim() || `git fetch exited ${String(fetched.code)}`),
+    };
+  }
+
+  for (const marker of markers) {
+    const resolved = await exec(
+      ["git", "--git-dir", mirror, "rev-parse", "--verify", `${marker}^{commit}`],
+      { env },
+    );
+    if (resolved.code !== 0) {
+      return {
+        state: "unknown",
+        error: `critical-base marker "${marker}" is not resolvable in the ${repo.name} mirror`,
+      };
+    }
+    const markerSha = resolved.stdout.trim();
+    if (markerSha === "") {
+      return { state: "unknown", error: `critical-base marker "${marker}" resolved to no commit` };
+    }
+
+    const localHas = await exec(
+      ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, branchRef],
+      { env },
+    );
+    if (localHas.code === 0) continue;
+
+    // The reattach source lacks the marker. If the live branch now carries it
+    // and the reattach source is its ancestor, the operator advanced the branch
+    // on GitHub (e.g. merged base into the PR branch): fold that fast-forward
+    // into the reattach source — safe, work-preserving — and accept the marker.
+    const tracked = `refs/remotes/origin/${branch}`;
+    const liveFetched = await exec(
+      ["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+refs/heads/${branch}:${tracked}`],
+      { env },
+    );
+    if (liveFetched.code !== 0) {
+      return {
+        state: "unknown",
+        error: scrubUserinfo(liveFetched.stderr.trim() || liveFetched.stdout.trim() || `git fetch ${branch} exited ${String(liveFetched.code)}`),
+      };
+    }
+    const liveHas = await exec(
+      ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", markerSha, tracked],
+      { env },
+    );
+    if (liveHas.code === 0) {
+      const localIsAncestor = await exec(
+        ["git", "--git-dir", mirror, "merge-base", "--is-ancestor", branchRef, tracked],
+        { env },
+      );
+      if (localIsAncestor.code === 0) {
+        await exec(["git", "--git-dir", mirror, "fetch", "--no-tags", "origin", `+${branchRef}:${branchRef}`], { env });
+        continue;
+      }
+    }
+
+    // Predates the marker and cannot be safely advanced to it. Name the base
+    // commits the branch has not absorbed so the hold's recovery is concrete.
+    const range: string[] = [];
+    const missing = await exec(
+      ["git", "--git-dir", mirror, "log", "--oneline", "--format=%h", `${branchRef}..${baseRef}`],
+      { env },
+    );
+    if (missing.code === 0) {
+      range.push(
+        ...missing.stdout
+          .trim()
+          .split("\n")
+          .filter((line) => line.length > 0)
+          .slice(0, CRITICAL_BASE_RANGE_MAX),
+      );
+    }
+    return { state: "stale", marker: markerSha, range };
+  }
+
+  return { state: "fresh" };
+}

package/src/lifecycle.ts

@@ -25,6 +25,10 @@ import { spawn, spawnSync } from "node:child_process";
 import { chmodSync, closeSync, mkdirSync, openSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
+// Type-only: erased at runtime, so the "free of every other module" property
+// below survives — this module still opens no store, loads no config and runs
+// no `gh`, and `stop`/`status` keep working when the config is broken.
+import type { DaemonStopDraft } from "./types.ts";
 
 /**
  * The systemd unit name operators are expected to install for a supervised
@@ -480,6 +484,58 @@ export async function startDaemon(
   );
 }
 
+/**
+ * The delivery facts a mediated stop/restart records at the exact chokepoint:
+ * the request's own provenance, how the stop is about to be delivered, and the
+ * daemon pid it targets (#378). The recorder is injected by the CLI callers
+ * (which own the store); this module stays free of it.
+ */
+export interface StopDelivery {
+  /** The request facts built by the CLI (commands/stop.ts, commands/restart.ts). */
+  provenance: DaemonStopDraft;
+  /** Whether the stop is about to go through `systemctl` or a raw signal. */
+  via: "systemctl" | "signal";
+  /** The daemon pid being stopped, when one was known. */
+  pid?: number;
+}
+
+/** The injected recorder invoked immediately before a stop/restart is signalled. */
+export type StopDeliveryFn = (stop: StopDelivery) => void;
+
+/**
+ * The honest provenance for a daemon that received a stop with no mediated
+ * request anywhere: unattributed by construction — Linux exposes no sender
+ * identity for a signal, so the record must say "unattributed" rather than
+ * guess one — plus what the receiving process actually knows: its own pid,
+ * the moment (stamped by the store), the runtime directory, and the projects
+ * it serves with their live-run counts.
+ *
+ * Storage belongs at the caller: the daemon's SIGINT/SIGTERM path writes the
+ * returned draft through the store before/while draining. This builds the
+ * facts, store-free, so a raw `kill`/out-of-band `systemctl stop` — no
+ * conductor CLI in the delivery path at all — still leaves a durable row the
+ * next `omp-conductor status` can show.
+ */
+export function externalStopProvenance(o: {
+  /** The daemon pid that received the signal, when known. */
+  daemonPid?: number;
+  /** The daemon's runtime directory (host paths only — never secrets). */
+  runtimeDir: string;
+  /** Every served project with its live-run count at signal time. */
+  affected: { project: string; live: number }[];
+  reason?: string;
+}): DaemonStopDraft {
+  return {
+    controlPath: "external signal",
+    scope: "global",
+    ...(o.daemonPid === undefined ? {} : { daemonPid: o.daemonPid }),
+    runtimeDir: o.runtimeDir,
+    affected: o.affected,
+    reason: o.reason ?? "external signal — no mediated stop request was recorded",
+    unattributed: true,
+  };
+}
+
 /**
  * How the last stop actually landed. Callers print this so an operator can
  * tell a supervised stop from a bare SIGTERM without reading the journal.
@@ -510,7 +566,35 @@ export interface RestartResult {
  * a stopped unit). The grace period is a deadline, not a clean drain: a tick
  * with a worker in flight can run for that worker's whole wall clock.
  */
-export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopResult> {
+export async function stopDaemon(
+  o: {
+    timeoutMs?: number;
+    /**
+     * The request facts for this stop, when a mediated caller carries them.
+     * With `record`, the durable row is written immediately before the signal
+     * is sent — the exact ordering "provenance precedes signalling" refers to
+     * — and only when a stop actually lands: a request that finds no daemon
+     * stops nothing and records nothing. Drain-style callers whose restart may
+     * never execute additionally record the request at entry themselves.
+     */
+    provenance?: DaemonStopDraft;
+    /** Invoked just before the daemon is signalled, with the delivery method. */
+    record?: StopDeliveryFn;
+  } = {},
+): Promise<StopResult> {
+  const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
+    if (o.provenance === undefined || o.record === undefined) return;
+    o.record({
+      provenance: {
+        ...o.provenance,
+        controlPath: `${o.provenance.controlPath} via ${via}`,
+        ...(pid === undefined ? {} : { daemonPid: pid }),
+      },
+      via,
+      pid,
+    });
+  };
+
   const rec = livingDaemon();
   if (rec === undefined) {
     // `livingDaemon` already cleared a stale file; this covers the unparseable
@@ -521,6 +605,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
       throw new Error(ownershipUnknown("stop", decision.reason));
     }
     if (decision.kind === "stop") {
+      recordStop("systemctl", decision.mainPid);
       await runSystemdStop(decision.mainPid, o.timeoutMs);
       clearRecord();
       return { kind: "stopped", pid: decision.mainPid, via: "systemctl" };
@@ -536,6 +621,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
     throw new Error(ownershipUnknown("stop", decision.reason));
   }
   if (decision.kind === "stop") {
+    recordStop("systemctl", rec.pid);
     await runSystemdStop(rec.pid, o.timeoutMs);
     clearRecord();
     return { kind: "stopped", pid: rec.pid, via: "systemctl" };
@@ -544,6 +630,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
   // decision.kind === "not-ours": confirmed no unit, inactive unit, no systemd
   // binary, or a unit whose MainPID is somebody else. Only a *confirmed*
   // negative is safe to signal.
+  recordStop("signal", rec.pid);
   const gone = await terminate(rec.pid, o.timeoutMs ?? STOP_TIMEOUT_MS);
   if (!gone) {
     // The record stays: something is still holding that pid, and forgetting
@@ -577,8 +664,29 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
  * Returns the record of the process that is now answering `/healthz`.
  */
 export async function restartDaemon(
-  o: { port?: number; project?: string; timeoutMs?: number } = {},
+  o: {
+    port?: number;
+    project?: string;
+    timeoutMs?: number;
+    /** Request facts and recorder, same semantics as {@link stopDaemon}: the
+     *  durable row is written immediately before the restarting signal. */
+    provenance?: DaemonStopDraft;
+    record?: StopDeliveryFn;
+  } = {},
 ): Promise<RestartResult> {
+  const recordStop = (via: "systemctl" | "signal", pid: number | undefined): void => {
+    if (o.provenance === undefined || o.record === undefined) return;
+    o.record({
+      provenance: {
+        ...o.provenance,
+        controlPath: `${o.provenance.controlPath} via ${via}`,
+        ...(pid === undefined ? {} : { daemonPid: pid }),
+      },
+      via,
+      pid,
+    });
+  };
+
   const previous = livingDaemon();
   const ownership = probeUnit();
   if (ownership.kind === "unknown") {
@@ -591,6 +699,7 @@ export async function restartDaemon(
     // startDaemon() — that leaves the unit failed while handing the operator
     // a healthy-looking but unmanaged pid (the 12:17Z incident in #376).
     // Any manager refusal is terminal; so is unproven ownership afterwards.
+    recordStop("systemctl", previous?.pid);
     const record = await restoreFailedUnit(o.timeoutMs, o.project);
     return { previous, record, via: "systemctl" };
   }
@@ -602,6 +711,7 @@ export async function restartDaemon(
     // Ownership is proven. A refused/timed-out restart must not fall through
     // to stopDaemon's signal path — that is the exact bounce this module exists
     // to prevent (SIGTERM → exit 143 → Restart=on-failure → new MainPID).
+    recordStop("systemctl", ownership.pid);
     const ran = systemctl(["restart", SYSTEMD_UNIT]);
     if (!ran.ok) {
       throw new Error(systemctlFailure("restart", ran));
@@ -615,6 +725,7 @@ export async function restartDaemon(
 
   // Confirmed unmanaged: no unit, an inactive unit, or a unit whose MainPID
   // is somebody else. The detached CLI daemon is the only path left.
+  recordStop("signal", previous?.pid);
   await stopDaemon({ timeoutMs: o.timeoutMs });
   const record = await startDaemon({ port: o.port ?? previous?.port, project: o.project ?? previous?.project });
   return { previous, record, via: "cli" };