omp-conductor 0.20.2 → 0.20.3

package/src/lifecycle.ts

@@ -194,6 +194,103 @@ export function livingDaemon(): DaemonRecord | undefined {
   return undefined;
 }
 
+/**
+ * `DaemonRecord.logFile` for a daemon nobody spawned — which is how systemd
+ * runs it (`ExecStart … omp-conductor daemon`) and how a hand-run foreground
+ * `daemon` works too. The field is required and `status` prints it, so it has
+ * to say something true: a foreground daemon opened no log of its own —
+ * whoever started it owns its stdout, be that the journal, a terminal, or a
+ * pane.
+ */
+const FOREGROUND_LOG = "<inherited stdout — started in the foreground>";
+
+/** What this process would publish as its own boot record, given its flags. */
+function foregroundRecord(o: { port?: number; project?: string }): DaemonRecord {
+  return {
+    pid: process.pid,
+    port: o.port ?? DEFAULT_PORT,
+    startedAt: Date.now(),
+    logFile: FOREGROUND_LOG,
+    ...(o.project === undefined ? {} : { project: o.project }),
+  };
+}
+
+export type ForegroundBoot =
+  | { kind: "claimed"; record: DaemonRecord }
+  | { kind: "foreign-live"; pid: number };
+
+/**
+ * The boot half of pidfile ownership (#1114): a daemon that starts without a
+ * managed spawn — systemd's `ExecStart`, a reboot, a bare hand-run — publishes
+ * its own record here, so the single-owner guard (`startDaemon`'s live-record
+ * refusal), `wake`, `status` and orphan reconciliation are armed no matter who
+ * started the process.
+ *
+ * A foreign live record is refused, never stolen; a stale one is cleared by
+ * {@link livingDaemon} on the way past; and a record already naming this pid
+ * was written by whoever spawned us and knows the real log file — replacing
+ * it with a guess would be a downgrade.
+ */
+export function publishForegroundRecord(
+  o: { port?: number; project?: string } = {},
+): ForegroundBoot {
+  const running = livingDaemon();
+  if (running !== undefined && running.pid !== process.pid) {
+    return { kind: "foreign-live", pid: running.pid };
+  }
+  if (running !== undefined) return { kind: "claimed", record: running };
+  const record = foregroundRecord(o);
+  writeRecord(record);
+  return { kind: "claimed", record };
+}
+
+/**
+ * The running half of pidfile ownership (#1114): the record can vanish under
+ * a perfectly healthy daemon — an external sweep of the shared runtime
+ * directory, a tmp cleaner — and nothing else re-arms the single-owner guard
+ * until the next restart. The daemon itself repairs that: absent or stale
+ * means it publishes again; a live foreign record is left alone (a `--once`
+ * drill may legitimately own visibility mid-tick).
+ */
+export function republishSelfRecord(o: { port?: number; project?: string } = {}): void {
+  const current = livingDaemon();
+  // Ours is already published, or a live foreign process owns visibility:
+  // either way there is nothing to repair.
+  if (current !== undefined) return;
+  writeRecord(foregroundRecord(o));
+}
+
+/** How often a running daemon re-asserts its record ({@link republishSelfRecord}). */
+export const RECORD_REPAIR_INTERVAL_MS = 60_000;
+
+/** Stops the periodic repair started by {@link startRecordRepair}. */
+export interface RecordRepair {
+  stop(): void;
+}
+
+/**
+ * Re-assert the record on an interval while the daemon runs. A failed repair
+ * never propagates: the pidfile is visibility, not correctness, and a failed
+ * attempt retries on the next tick rather than killing the dispatcher over it.
+ */
+export function startRecordRepair(
+  o: { port?: number; project?: string } = {},
+  intervalMs = RECORD_REPAIR_INTERVAL_MS,
+): RecordRepair {
+  const timer = setInterval(() => {
+    try {
+      republishSelfRecord(o);
+    } catch (err) {
+      process.stderr.write(
+        `omp-conductor: could not re-assert the daemon pidfile: ${(err as Error).message}\n`,
+      );
+    }
+  }, intervalMs);
+  // Bookkeeping must never hold the event loop open at shutdown.
+  timer.unref?.();
+  return { stop: () => clearInterval(timer) };
+}
+
 /**
  * The `daemon --once` lease file, one per process. Created `O_EXCL` so two
  * simultaneous `--once` runs cannot both win ownership, and it names the
@@ -1186,6 +1283,10 @@ export type SystemctlResult = {
   stderr: string;
   /** Binary not found (ENOENT). Distinct from a failed invocation of a present binary. */
   missing?: boolean;
+  /** The client bound expired (spawnSync ETIMEDOUT). A state-changing verb
+   *  that reports this is re-judged from the unit's observed state before it
+   *  becomes a failure (#1112). */
+  timedOut?: boolean;
 };
 
 export type SystemctlFn = (args: string[]) => SystemctlResult;
@@ -1251,14 +1352,169 @@ function testSystemctl(args: string[]): SystemctlResult {
   }
 }
 
-function defaultSystemctl(args: string[]): SystemctlResult {
-  if (process.env["NODE_ENV"] === "test") return testSystemctl(args);
+/** Verbs whose completion is a unit job: systemd holds the client until the
+ * stop/start finishes, which legitimately takes as long as the unit's own
+ * graceful-shutdown budget. Everything else answers from manager state and
+ * keeps the short probe bound (#1112). */
+const STATE_CHANGING_VERBS: Record<string, true> = {
+  start: true,
+  stop: true,
+  restart: true,
+  "try-restart": true,
+  reload: true,
+  "reload-or-restart": true,
+};
+
+/**
+ * Bound for probe verbs (`show`, `is-active`, `is-enabled`, …): a hung dbus
+ * becomes `unknown` ownership instead of a hang. Callers must NOT treat that
+ * timeout as "no unit". State-changing verbs get their own budget below — a
+ * 15s bound on `restart` is what declared a healthy restart failed and rolled
+ * a good release back (#1112).
+ */
+const SYSTEMCTL_PROBE_TIMEOUT_MS = 15_000;
+
+/**
+ * systemd's built-in `TimeoutStopSec`: how long any unit without an explicit
+ * override may legitimately spend stopping before systemd itself escalates.
+ * A client bound for a state-changing verb never sits below this floor —
+ * smaller is structurally able to lose against a healthy, graceful stop.
+ */
+const SYSTEMD_DEFAULT_TIMEOUT_STOP_MS = 90_000;
+
+/** Slack over the unit's own stop budget: the start half of a restart plus
+ * scheduling delay, before the client's stopwatch means anything at all. */
+const STATE_VERB_SLACK_MS = 30_000;
+
+type RawSystemctl = (args: string[], timeoutMs: number) => SystemctlResult;
+
+/** One unit-state reading, in the shape the post-timeout verdict needs. */
+interface UnitStateSample {
+  activeState?: string;
+  /** `ExecMainStartTimestampMonotonic` — changes on every fresh start of the
+   *  unit, which is what makes "the old instance" distinguishable. */
+  startStampMonotonic?: string;
+  /** `TimeoutStopUSec` — integer microseconds, or `infinity`. */
+  timeoutStopUSec?: string;
+}
+
+const STATE_SAMPLE_PROPERTIES = [
+  "--property=ActiveState",
+  "--property=ExecMainStartTimestampMonotonic",
+  "--property=TimeoutStopUSec",
+] as const;
+
+function sampleUnitState(unit: string, raw: RawSystemctl): UnitStateSample | undefined {
+  const ran = raw(
+    ["show", unit, "--value", ...STATE_SAMPLE_PROPERTIES],
+    SYSTEMCTL_PROBE_TIMEOUT_MS,
+  );
+  if (!ran.ok) return undefined;
+  const values = ran.stdout.split("\n");
+  const pick = (index: number): string | undefined => {
+    const value = values[index]?.trim();
+    return value === undefined || value === "" ? undefined : value;
+  };
+  return {
+    activeState: pick(0),
+    startStampMonotonic: pick(1),
+    timeoutStopUSec: pick(2),
+  };
+}
+
+/**
+ * The client bound for one state-changing verb: at least the unit's own
+ * `TimeoutStopUSec` — read live, because any fixed constant re-creates this
+ * bug on whatever host drains slower than it — and never below systemd's own
+ * default stop budget. An unreadable budget gets the documented fallback.
+ */
+function stateVerbTimeoutMs(sample: UnitStateSample | undefined): number {
+  let stopMs = SYSTEMD_DEFAULT_TIMEOUT_STOP_MS;
+  const usec =
+    sample?.timeoutStopUSec === undefined ? Number.NaN : Number.parseInt(sample.timeoutStopUSec, 10);
+  if (Number.isFinite(usec) && usec > 0) {
+    stopMs = Math.max(Math.ceil(usec / 1000), stopMs);
+  }
+  return stopMs + STATE_VERB_SLACK_MS;
+}
+
+/**
+ * The stopwatch lost; the unit's own state decides. A state-changing verb that
+ * outlived the client bound succeeded exactly when the manager shows the
+ * verb's goal reached: not-active for `stop`, active with a FRESH start
+ * timestamp for the start/restart family — an unchanged timestamp means the
+ * old instance is still the running one. Anything else stays a failure that
+ * carries both facts.
+ */
+function stateVerbTimeoutVerdict(
+  verb: string,
+  unit: string,
+  before: UnitStateSample | undefined,
+  after: UnitStateSample | undefined,
+  timedOut: SystemctlResult,
+  timeoutMs: number,
+): SystemctlResult {
+  const base =
+    `systemctl ${verb} ${unit} exceeded its ${Math.round(timeoutMs / 1000)}s client bound ` +
+    `(${timedOut.stderr.trim() || "client timeout"})`;
+  if (after?.activeState === undefined) {
+    return {
+      ...timedOut,
+      stderr:
+        `${base}, and the unit's state could not be re-read afterwards — ` +
+        `the outcome is unproven, not failed`,
+    };
+  }
+  if (verb === "stop") {
+    if (after.activeState !== "active" && after.activeState !== "deactivating") {
+      // Goal state reached: the stop completed, whatever the stopwatch said.
+      return { ok: true, stdout: "", stderr: "" };
+    }
+    return {
+      ...timedOut,
+      stderr: `${base}; the unit is still ${after.activeState} — the stop did not complete`,
+    };
+  }
+  const freshStart =
+    before !== undefined &&
+    after.startStampMonotonic !== undefined &&
+    after.startStampMonotonic !== before.startStampMonotonic;
+  if (after.activeState === "active" && freshStart) {
+    // The unit restarted; the client simply gave up first.
+    return { ok: true, stdout: "", stderr: "" };
+  }
+  const why =
+    after.activeState !== "active"
+      ? `the unit is ${after.activeState}`
+      : before === undefined
+        ? "the unit is active but no pre-call sample exists to prove the start is fresh"
+        : "its start timestamp is unchanged, so the running instance predates the call";
+  return { ...timedOut, stderr: `${base}; ${why} — the ${verb} did not complete` };
+}
+
+/**
+ * Run one systemctl invocation with #1112 semantics: probes keep the short
+ * hung-dbus bound, state-changing verbs wait out the unit's own stop budget,
+ * and a state-changing verb that outlives even that is judged by the unit's
+ * observed state rather than the client's stopwatch. Exported for tests,
+ * which inject their own {@link RawSystemctl}.
+ */
+export function runSystemctlInvocation(args: string[], raw: RawSystemctl): SystemctlResult {
+  const verb = args[0] ?? "";
+  if (STATE_CHANGING_VERBS[verb] !== true) return raw(args, SYSTEMCTL_PROBE_TIMEOUT_MS);
+  const unit = args.length > 1 && !args[1]!.startsWith("-") ? args[1]! : SYSTEMD_UNIT;
+  const before = sampleUnitState(unit, raw);
+  const timeoutMs = stateVerbTimeoutMs(before);
+  const ran = raw(args, timeoutMs);
+  if (ran.ok || !ran.timedOut) return ran;
+  return stateVerbTimeoutVerdict(verb, unit, before, sampleUnitState(unit, raw), ran, timeoutMs);
+}
+
+function spawnSystemctl(args: string[], timeoutMs: number): SystemctlResult {
   try {
     const res = spawnSync("systemctl", args, {
       encoding: "utf8",
-      // Bound the wait: a hung dbus becomes `unknown` ownership, not a hang.
-      // Callers must NOT treat that timeout as "no unit".
-      timeout: 15_000,
+      timeout: timeoutMs,
       env: process.env,
     });
     if (res.error) {
@@ -1268,6 +1524,7 @@ function defaultSystemctl(args: string[]): SystemctlResult {
         stdout: "",
         stderr: err.message,
         ...(err.code === "ENOENT" ? { missing: true } : {}),
+        ...(err.code === "ETIMEDOUT" ? { timedOut: true } : {}),
       };
     }
     return {
@@ -1280,6 +1537,11 @@ function defaultSystemctl(args: string[]): SystemctlResult {
   }
 }
 
+function defaultSystemctl(args: string[]): SystemctlResult {
+  if (process.env["NODE_ENV"] === "test") return testSystemctl(args);
+  return runSystemctlInvocation(args, spawnSystemctl);
+}
+
 let systemctl: SystemctlFn = defaultSystemctl;
 
 /** Shared manager runner. Production reaches the hardcoded binary; test

package/src/ready-gate.ts

@@ -95,6 +95,65 @@ export function acceptanceCriteriaSection(text: string): { heading: string; crit
   return { heading: lines[start]!.trim(), criteria };
 }
 
+/** The leading run of backticked tokens in a bullet: `` `a.ts`, `b/c.ts` ``
+ *  reads as two paths, while a backticked token after prose is prose, never a
+ *  declaration. Mirrors admission's private helper of the same shape — the
+ *  gate cannot import it, and a second grammar beside the first would make
+ *  issues that render correctly for one parse wrongly for the other. */
+function leadingBacktickedRun(text: string): string[] {
+  const trimmed = text.trim();
+  if (!trimmed.startsWith("`")) return [];
+  const run: string[] = [];
+  let pos = 0;
+  while (true) {
+    if (trimmed[pos] !== "`") break;
+    const close = trimmed.indexOf("`", pos + 1);
+    if (close < 0) break;
+    const token = trimmed.slice(pos + 1, close).trim();
+    if (token === "") break;
+    run.push(token);
+    pos = close + 1;
+    const sep = /^[,\s]+/.exec(trimmed.slice(pos));
+    if (sep === null) break;
+    pos += sep[0].length;
+  }
+  return run;
+}
+
+/**
+ * The read-only entry points an issue declares: files its worker must read —
+ * the caller to understand, the file that proves the convention — but never
+ * write (#1108). The grammar mirrors the write-lane section: a `#`-to-`######`
+ * heading spelling `Read-only` optionally followed by `entry points`, an
+ * optional trailing colon, then the contiguous bullet run any non-bullet line
+ * ends; each bullet contributes its leading backticked run, so an issue that
+ * renders correctly for the lane sections renders correctly here too. Unlike
+ * the lane there is no inline form and no bare-token fallback: marking a file
+ * read-only is a deliberate act, and the backticked bullet is the whole shape.
+ *
+ * Every occurrence in the text counts, where a lane declaration replaces:
+ * read-only lists accumulate, so a correction comment adds references without
+ * restating the body's.
+ */
+export function readOnlyEntryPoints(text: string): string[] {
+  const files: string[] = [];
+  let inSection = false;
+  for (const line of text.split("\n")) {
+    if (/^[ \t]*#{1,6}[ \t]+read[-\s]only(?:[ \t]+entry[ \t]+points)?[ \t]*[:.]?[ \t]*$/i.test(line)) {
+      inSection = true;
+      continue;
+    }
+    if (!inSection || line.trim() === "") continue;
+    const marker = line.match(/^[ \t]*(?:[-*+]|\d+[.)])[ \t]+(?:\[[ xX]\][ \t]+)?/);
+    if (marker === null) {
+      inSection = false;
+      continue;
+    }
+    files.push(...leadingBacktickedRun(line.slice(marker[0].length)));
+  }
+  return files;
+}
+
 /** Trimmed non-empty strings out of a value the type system claims is a string
  *  array. Defensive because the gate is the last thing between a hand-edited
  *  store row and a dispatched worker: a malformed field is a refusal, never a
@@ -114,7 +173,7 @@ function strings(value: unknown): string[] {
  * May this verdict be promoted mechanically?
  *
  * Every check is one named miss. The order is fixed — verdict, readability,
- * acceptance criteria, write lane, proof commands, sizing evidence,
+ * acceptance criteria, write lane, entry points, proof commands, sizing evidence,
  * dependencies, premise, routing, state labels — so the same input always
  * produces the same list and a digest line is stable across ticks.
  *
@@ -181,6 +240,7 @@ export function readyGate(input: ReadyGateInput): ReadyGateVerdict {
       ? undefined
       : (writeLaneSectionHeading(body) ??
         comments.map((comment) => writeLaneSectionHeading(comment.body)).find((found) => found !== undefined));
+  const declared = lane === undefined ? [] : [...new Set(strings(lane.files))].sort();
   if (verdictLane.length === 0) {
     // The schema requires at least one path, so this is a hand-edited or
     // corrupt row. An empty lane on both sides would otherwise *match*, and
@@ -190,11 +250,46 @@ export function readyGate(input: ReadyGateInput): ReadyGateVerdict {
     missing.push(
       `the write-lane section (${heading}) parsed no path-like files — name them as backticked bullets directly under the heading`,
     );
-  } else {
-    const declared = lane === undefined ? [] : [...new Set(strings(lane.files))].sort();
-    if (declared.length !== verdictLane.length || declared.some((path, index) => path !== verdictLane[index])) {
+  } else if (declared.length !== verdictLane.length || declared.some((path, index) => path !== verdictLane[index])) {
+    missing.push(
+      `the issue's write lane [${declared.join(", ")}] disagrees with the verdict's file lane [${verdictLane.join(", ")}]`,
+    );
+  }
+
+  // An entry point outside the lane is usually legitimate — the reference
+  // file, the caller to understand first — so the gate refuses the *unstated*
+  // one, not the idea (#1108): eight settlement flags in one day were lane
+  // escapes, several into files the issue itself had already named as an
+  // entry point. Putting the file in the lane it is meant to be written in,
+  // or listing it under a `## Read-only entry points` heading, both satisfy
+  // the check; a hard refusal for every reference would push promoters into
+  // stopping the one field workers most need.
+  //
+  // Judged only when the two lane lists agree: a disagreement is refused
+  // anyway, and entry points computed against a lane about to be rewritten
+  // would be noise stacked on the actionable fact. No parsed lane means
+  // nothing to stand outside of.
+  const entryPoints = [...new Set(strings(result.entryPoints))];
+  const lanesAgree =
+    verdictLane.length > 0 &&
+    declared.length === verdictLane.length &&
+    declared.every((path, index) => path === verdictLane[index]);
+  if (entryPoints.length > 0 && lanesAgree) {
+    const readOnly = [
+      ...new Set([body, ...comments.map((comment) => comment.body)].flatMap((text) => readOnlyEntryPoints(text))),
+    ];
+    // Exact member or anything under a trailing-slash directory entry — the
+    // containment the lane enforcement itself applies to written files
+    // (diff-flags.ts:withinLane).
+    const escaped = entryPoints.filter(
+      (path) =>
+        !declared.includes(path) &&
+        !declared.some((entry) => entry.endsWith("/") && path.startsWith(entry)) &&
+        !readOnly.includes(path),
+    );
+    if (escaped.length > 0) {
       missing.push(
-        `the issue's write lane [${declared.join(", ")}] disagrees with the verdict's file lane [${verdictLane.join(", ")}]`,
+        `entry points ${escaped.join(", ")} are neither in the issue's write lane [${declared.join(", ")}] nor marked read-only — put each in the \`## Exact write lane\` bullets meant for writing, or declare it under \`## Read-only entry points\``,
       );
     }
   }

package/src/upgrade.ts

@@ -1696,7 +1696,8 @@ export async function upgradeConductor(
     journal({ kind: "phase", phase: "drain", ok: false, version: release.version, detail: err instanceof Error ? err.message : String(err) });
     const failure = err instanceof Error ? err.message : String(err);
     throw new Error(
-      `upgrade failed while draining workers: ${failure}; no installation started; dispatch remains paused`,
+      `upgrade failed while draining workers: ${failure}; no installation started; ` +
+        `dispatch remains paused — resume dispatch with \`omp-conductor resume --all\``,
     );
   }
   journal({ kind: "phase", phase: "drain", ok: true, version: release.version });
@@ -1916,7 +1917,8 @@ export async function upgradeConductor(
           `Restored: cli=${surfaces.cliVersion}, omp=${surfaces.ompVersion ?? "missing"}, ` +
             `herdr=${surfaces.herdrSource ?? "missing"}`,
           ...(preUpgradeBackup === undefined ? [] : [`Pre-upgrade config snapshot: ${preUpgradeBackup}`]),
-          "Dispatch stays paused until the first tick after the restart records the rollback verified.",
+          "Dispatch stays paused until the first tick after the restart records the rollback verified; " +
+            "if it is still paused, resume it with `omp-conductor resume --all`.",
         ],
         `upgrade:${release.version}:rolled-back`,
       );
@@ -1931,16 +1933,21 @@ export async function upgradeConductor(
           `Failure: ${failure}`,
           installAttribution,
           `Rollback failure: ${rollback}`,
-          "The fleet may be at mixed versions; do not resume dispatch.",
+          "The fleet may be at mixed versions; do not resume dispatch until versions are verified.",
+          "Once one version is confirmed everywhere, `omp-conductor resume --all` lifts the pause.",
           "Restore manually from the journal at " + upgradeJournalPath(stateDir()) + ".",
         ],
         `upgrade:${release.version}:rollback-failed`,
       );
       throw new Error(
-        `upgrade failed: ${failure}; rollback incomplete: ${rollback}; dispatch remains paused`,
+        `upgrade failed: ${failure}; rollback incomplete: ${rollback}; dispatch remains paused — ` +
+          `\`omp-conductor resume --all\` lifts the pause once mixed versions are ruled out`,
       );
     }
-    throw new Error(`upgrade failed: ${failure}; previous installation restored; dispatch remains paused`);
+    throw new Error(
+      `upgrade failed: ${failure}; previous installation restored; dispatch remains paused — ` +
+        `resume dispatch with \`omp-conductor resume --all\``,
+    );
   }
 
   // The install landed; name any host-runtime destination this version no