omp-conductor 0.15.12 → 0.15.13

package/REFERENCE.md

@@ -2633,10 +2633,15 @@ Known and deliberate in this version:
   restart policy stays a visible detector rather than a silent spin. Prefer
   `omp-conductor stop` / `systemctl stop` over raw `kill`: a raw kill is not an
   intentional stop, and the daemon comes straight back.
-- **A failed orchestrator degrades quietly.** The daemon logs a warning and keeps
-  running, but tier-1 escalations then land in issue comments — which is exactly the
-  "nobody reads it until morning" path the orchestrator exists to avoid. The warning
-  is in `daemon.log`; nothing pages you about it.
+- **A failed orchestrator doesn't stay quiet.** Start failure and an
+  unexpected session death each open one durable orchestrator-down incident
+  that pages tier 2 once ("down since <t>, tier-1 escalations diverting to
+  issue comments") and is shown as a degrade row by `omp-conductor status` —
+  mode, since-when, and how many tier-1 escalations were diverted while it was
+  down. The incident survives a daemon restart while still down (it is
+  re-derived, not forgotten), and one closing notice lands when the orchestrator
+  recovers. What remains a separate watchdog is the `.conductor-stalled` wedge
+  path, for a session that stays alive but stops draining its queue.
 - **Workers are not terminal panes, so you cannot watch them there.** Each
   worker is an omp session the daemon starts as a child process. The resident
   daemon tracks workers in a background pool so the five-minute loop keeps

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "omp-conductor",
-  "version": "0.15.12",
+  "version": "0.15.13",
   "type": "module",
   "license": "MIT",
   "description": "A 24/7 dispatcher that takes ready GitHub issues to green, mergeable PRs using omp coding sessions, with tiered escalation first to an orchestrator session and then to a human.",

package/src/commands/restart.ts

@@ -7,70 +7,97 @@
  */
 
 import type { CommandContext } from "./context.ts";
+import { buildStopProvenance } from "../stop-provenance.ts";
 import { setPaused } from "../daemon.ts";
-import { restartDaemon, type RestartResult } from "../lifecycle.ts";
+import { restartDaemon, type RestartResult, type StopDelivery } from "../lifecycle.ts";
+import { openStore, dbPath } from "../store.ts";
 import { DEFAULT_DEPS, drainAndRestart, type UpgradeDeps } from "../upgrade.ts";
 
 export async function restartCommand(ctx: CommandContext): Promise<void> {
-if (ctx.argv.includes("--now")) {
-  // Inherit the running daemon's port and project: a restart that quietly
-  // moved to the default port would leave every existing health check
-  // pointing at nothing. When the unit owns the live pid, restartDaemon
-  // goes through systemctl so the replacement stays supervised; a failed
-  // installed unit is reset and started through systemd the same way and
-  // is never replaced by an unmanaged daemon. Success is only reported
-  // after the manager is proven to own the reported pid.
-  const { previous, record, via } = await restartDaemon({
-    port: ctx.portFlag(),
-    project: ctx.projectFlag,
+const store = openStore(dbPath());
+try {
+  // Provenance (#378): one record naming the request, every configured
+  // project with its live-run count, and — via the lifecycle chokepoint — the
+  // delivery method used. The shared daemon serves all projects, so a restart
+  // is never silent even when the operator narrows nothing.
+  const provenance = buildStopProvenance({
+    controlPath: "cli restart",
+    reason: "operator restart",
+    scope: "global",
   });
-  if (previous !== undefined) {
-    process.stdout.write(
-      `stopped — pid ${previous.pid}${via === "systemctl" ? " (via systemctl)" : ""}\n`,
-    );
-  }
-  process.stdout.write(
-    `started — pid ${record.pid}, /healthz on :${record.port}` +
-      `${record.project === undefined ? "" : `, project ${record.project}`}` +
-      `${via === "systemctl" ? " (via systemctl)" : ""}\nlog ${record.logFile}\n`,
-  );
-  return;
-}
+  const record = (signal: StopDelivery) => store.recordDaemonStop(signal.provenance);
 
-// Default: drain the fleet before restarting — pause new claims, wait for
-// live workers to reach 0/N (bounded by --timeout), restart, then restore
-// the prior dispatch state. A timed-out drain restarts nothing and leaves
-// dispatch paused, so it is safe to run while the daemon is wedged.
-const timeoutRaw = ctx.flag("timeout");
-const timeoutSeconds =
-  timeoutRaw === undefined ? 1800 : Number.parseInt(timeoutRaw, 10);
-const timeoutMs = (Number.isNaN(timeoutSeconds) ? 1800 : timeoutSeconds) * 1000;
-let result!: RestartResult;
-const deps: UpgradeDeps = {
-  ...DEFAULT_DEPS,
-  setPaused: (v, project) =>
-    setPaused(v, { source: "restart", reason: "restart, draining" }, project),
-  restartDaemon: async () => {
-    result = await restartDaemon({
+  if (ctx.argv.includes("--now")) {
+    // Inherit the running daemon's port and project: a restart that quietly
+    // moved to the default port would leave every existing health check
+    // pointing at nothing. When the unit owns the live pid, restartDaemon
+    // goes through systemctl so the replacement stays supervised; a failed
+    // installed unit is reset and started through systemd the same way and
+    // is never replaced by an unmanaged daemon. Success is only reported
+    // after the manager is proven to own the reported pid.
+    const { previous, record: newRecord, via } = await restartDaemon({
       port: ctx.portFlag(),
       project: ctx.projectFlag,
+      provenance,
+      record,
     });
-  },
-};
-try {
-  await drainAndRestart(deps, { project: ctx.projectFlag, timeoutMs });
-} catch (err) {
-  process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
-  process.exit(1);
-}
-if (result.previous !== undefined) {
+    if (previous !== undefined) {
+      process.stdout.write(
+        `stopped — pid ${previous.pid}${via === "systemctl" ? " (via systemctl)" : ""}\n`,
+      );
+    }
+    process.stdout.write(
+      `started — pid ${newRecord.pid}, /healthz on :${newRecord.port}` +
+        `${newRecord.project === undefined ? "" : `, project ${newRecord.project}`}` +
+        `${via === "systemctl" ? " (via systemctl)" : ""}\nlog ${newRecord.logFile}\n`,
+    );
+    return;
+  }
+
+  // Default: drain the fleet before restarting — pause new claims, wait for
+  // live workers to reach 0/N (bounded by --timeout), restart, then restore
+  // the prior dispatch state. A timed-out drain restarts nothing and leaves
+  // dispatch paused, so it is safe to run while the daemon is wedged.
+  const timeoutRaw = ctx.flag("timeout");
+  const timeoutSeconds =
+    timeoutRaw === undefined ? 1800 : Number.parseInt(timeoutRaw, 10);
+  const timeoutMs = (Number.isNaN(timeoutSeconds) ? 1800 : timeoutSeconds) * 1000;
+  let result!: RestartResult;
+  const deps: UpgradeDeps = {
+    ...DEFAULT_DEPS,
+    setPaused: (v, project) =>
+      setPaused(v, { source: "restart", reason: "restart, draining" }, project),
+    restartDaemon: async () => {
+      result = await restartDaemon({
+        port: ctx.portFlag(),
+        project: ctx.projectFlag,
+        provenance,
+        record,
+      });
+    },
+  };
+  // Record the request at ENTRY, before the drain: a caller that disconnects
+  // or hits --timeout while workers drain never reaches the lifecycle
+  // chokepoint, and the durable provenance must survive that — it is exactly
+  // the stop-with-no-witness incident #378 exists for (#378).
+  store.recordDaemonStop(provenance);
+  try {
+    await drainAndRestart(deps, { project: ctx.projectFlag, timeoutMs });
+  } catch (err) {
+    process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
+    process.exit(1);
+  }
+  if (result.previous !== undefined) {
+    process.stdout.write(
+      `stopped — pid ${result.previous.pid}${result.via === "systemctl" ? " (via systemctl)" : ""}\n`,
+    );
+  }
   process.stdout.write(
-    `stopped — pid ${result.previous.pid}${result.via === "systemctl" ? " (via systemctl)" : ""}\n`,
+    `started — pid ${result.record.pid}, /healthz on :${result.record.port}` +
+      `${result.record.project === undefined ? "" : `, project ${result.record.project}`}` +
+      `${result.via === "systemctl" ? " (via systemctl)" : ""}\nlog ${result.record.logFile}\n`,
   );
+} finally {
+  store.close();
 }
-process.stdout.write(
-  `started — pid ${result.record.pid}, /healthz on :${result.record.port}` +
-    `${result.record.project === undefined ? "" : `, project ${result.record.project}`}` +
-    `${result.via === "systemctl" ? " (via systemctl)" : ""}\nlog ${result.record.logFile}\n`,
-);
 }

package/src/commands/stop.ts

@@ -7,8 +7,10 @@
  */
 
 import type { CommandContext } from "./context.ts";
+import { buildStopProvenance } from "../stop-provenance.ts";
 import { hold, pinPaneHalt, stopConductorPane } from "../fleet.ts";
 import { stopDaemon } from "../lifecycle.ts";
+import { openStore, dbPath } from "../store.ts";
 
 export async function stopCommand(ctx: CommandContext): Promise<void> {
 const withPane = ctx.argv.includes("--pane");
@@ -24,28 +26,49 @@ const targets = ctx.targetProjects().map((project) => {
     pin: withPane ? pinPaneHalt(project.name).path : undefined,
   };
 });
-const stop = await stopDaemon();
-const stopLine =
-  stop.kind === "not-running"
-    ? "daemon was not running"
-    : `daemon stopped — pid ${stop.pid}${stop.via === "systemctl" ? " (via systemctl)" : ""}`;
-for (const target of targets) {
-  if (target.pin !== undefined) {
-    const pane = await stopConductorPane(target.project.name);
-    process.stdout.write(
-      `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
-        `${stopLine}\n` +
-        `pane recovery pinned at ${target.pin}\n` +
-        `pane stop: ${pane.stopped} — ${pane.detail}\n` +
-        `  (conductor agent "${pane.agentName}" only — herdr-fleet.service was NOT stopped;\n` +
-        `   resume clears the pin when you want recovery again)\n`,
-    );
-  } else {
-    process.stdout.write(
-      `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
-        `${stopLine}\n` +
-        `pane left running (pass --pane to stop the conductor agent and pin recovery)\n`,
-    );
+const store = openStore(dbPath());
+try {
+  // Provenance (#378): the shared daemon serves every configured project, so
+  // the record names the request's own project (or the global scope) AND every
+  // sibling with its live-run count — no silent cross-project stop.
+  const provenance = buildStopProvenance({
+    controlPath: "cli stop",
+    reason: "operator stop",
+    scope: targets.length > 1 ? "global" : "project",
+    project: targets.length === 1 ? targets[0]!.project.name : undefined,
+  });
+  const stop = await stopDaemon({
+    provenance,
+    // The durable row is written by the lifecycle at the exact chokepoint —
+    // immediately before systemctl/SIGTERM, with the delivery method that is
+    // about to be used — which is what makes "provenance precedes signalling"
+    // provable rather than claimed.
+    record: (signal) => store.recordDaemonStop(signal.provenance),
+  });
+  const stopLine =
+    stop.kind === "not-running"
+      ? "daemon was not running"
+      : `daemon stopped — pid ${stop.pid}${stop.via === "systemctl" ? " (via systemctl)" : ""}`;
+  for (const target of targets) {
+    if (target.pin !== undefined) {
+      const pane = await stopConductorPane(target.project.name);
+      process.stdout.write(
+        `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
+          `${stopLine}\n` +
+          `pane recovery pinned at ${target.pin}\n` +
+          `pane stop: ${pane.stopped} — ${pane.detail}\n` +
+          `  (conductor agent "${pane.agentName}" only — herdr-fleet.service was NOT stopped;\n` +
+          `   resume clears the pin when you want recovery again)\n`,
+      );
+    } else {
+      process.stdout.write(
+        `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
+          `${stopLine}\n` +
+          `pane left running (pass --pane to stop the conductor agent and pin recovery)\n`,
+      );
+    }
   }
+} finally {
+  store.close();
 }
 }