omp-conductor 0.7.1 → 0.9.0

package/src/cli.ts

@@ -8,7 +8,7 @@
 import { closeSync, openSync, readFileSync, readSync, statSync } from "node:fs";
 import { userInfo } from "node:os";
 import { dirname, join } from "node:path";
-import { runBoard } from "./board.ts";
+import { boardJson, boardSnapshotOnce, runBoard } from "./board.ts";
 import {
   applyRetrofit,
   formatBriefReport,
@@ -44,6 +44,7 @@ import {
   startDaemon,
   stopDaemon,
   writeRecord,
+  type RestartResult,
 } from "./lifecycle.ts";
 import { STALL_MARKER_FILE } from "./orchestrator-tick.ts";
 import { digestDedupeKey } from "./reports.ts";
@@ -63,7 +64,7 @@ import { handleVerbCall, type VerbChannel } from "./verbs/server.ts";
 import { REPORT_KINDS, VERB_NAMES } from "./types.ts";
 import type { ProjectConfig, ReportKind } from "./types.ts";
 import { formatUnblock, unblockIssue } from "./unblock.ts";
-import { upgradeConductor } from "./upgrade.ts";
+import { DEFAULT_DEPS, drainAndRestart, upgradeConductor, type UpgradeDeps } from "./upgrade.ts";
 
 const FRICTION_FEEDBACK_KINDS = {
   "escalation-digest": "feedback:escalation-should-digest",
@@ -89,9 +90,9 @@ usage:
   omp-conductor start [--port N] [--project NAME]
   omp-conductor --version
   omp-conductor stop
-  omp-conductor restart [--port N] [--project NAME]
+  omp-conductor restart [--now] [--timeout SECONDS] [--port N] [--project NAME]
   omp-conductor upgrade [--to VERSION] [--project NAME]
-  omp-conductor board [--project NAME]
+  omp-conductor board [--project NAME] [--json]
   omp-conductor status [--project NAME]
   omp-conductor ledger [--issue N] [--limit N] [--project NAME]
   omp-conductor hold [--project NAME]
@@ -101,10 +102,10 @@ usage:
   omp-conductor release-pane [--project NAME]
   omp-conductor tail <issue> [--project NAME]
   omp-conductor extend <issue> --turns N [--project NAME]
-  omp-conductor unblock <issue> [--force] [--project NAME]
+  omp-conductor unblock <issue> [--force] [--no-requeue] [--project NAME]
   omp-conductor verb <conductor_*> [--project NAME] [--arg k=v ...]
   omp-conductor daemon [--once] [--port N] [--project NAME]
-  omp-conductor pause
+  omp-conductor pause [--reason TEXT]
   omp-conductor resume
   omp-conductor graph-setup [--project NAME] [--write]
   omp-conductor brief-upgrade [--migrate|--retrofit] [--apply] [--file PATH] [--project NAME]
@@ -126,17 +127,23 @@ usage:
   stop     stop the running daemon. Uses systemctl when the omp-conductor
            unit owns the process (so Restart=on-failure cannot bring it back);
            otherwise SIGTERM then SIGKILL.
-  restart  stop then start, keeping the running daemon's port and project
-           unless a flag overrides them. On boot the new process salvages any
-           dirty live worktrees before orphaning those rows — see README
-           "Deploying a new package onto a busy fleet". Goes through systemctl
-           when the unit owns the live pid.
+  restart  drain the fleet by default: pause new claims, wait until live workers
+           reach 0/N (bounded by --timeout SECONDS, default 1800), restart keeping
+           the running daemon's port and project unless a flag overrides them,
+           then restore the prior dispatch state. --now skips the drain and
+           restarts immediately, orphaning any live runs (old behavior). A drain
+           that hits --timeout restarts nothing and leaves dispatch paused. On boot
+           the new process salvages dirty live worktrees before orphaning those
+           rows — see README "Deploying a new package onto a busy fleet". Goes
+           through systemctl when the unit owns the live pid.
   status   layered fleet report: dispatch (running|paused|stopped), ticks and
            next due time, pane, herdr, Telegram bot/API health, daemon, caps
            and active runs.
   board    open the live keyboard-driven fleet board. It renders queue holds,
            every run lifecycle stage, recent merges, spend and health; Enter
-           follows a selected transcript without leaving the board.
+           follows a selected transcript without leaving the board. --json (or
+           a non-interactive stdin/stdout) prints a one-shot JSON snapshot of
+           the same lanes instead.
   ledger   every conductor-verb call and how the daemon decided it: the verb,
            the arguments, allow or refuse, the named refusal reason, and the
            resulting sha. Sessions cannot push, open, merge, label or release
@@ -170,7 +177,10 @@ usage:
            operational-continuation budget, not failed implementation attempts.
            Refuses when the newest attempt's work could not be committed and
            its worktree is the only copy: re-claiming removes that tree. Use
-           --force once you have recovered it or accepted the loss.
+           --force once you have recovered it or accepted the loss. Restores
+           the project queue label by default so the issue is actually
+           dispatchable; --no-requeue clears state labels only (the "about to
+           close it" case).
   verb     run one conductor_* verb as the orchestrator, from the CLI: the same
            checks and the same ledger rows a session's call would get. This is
            how an external orchestrator merges, labels, releases or reads PR
@@ -202,6 +212,7 @@ usage:
   daemon   run the dispatch loop in the foreground; --once runs a single tick
            and exits. This is what \`start\` launches.
   pause    stop claiming new work only (ticks keep firing if armed). Prefer hold.
+           --reason TEXT is recorded in the pause sentinel (\`status\` shows it).
   resume   clear pause only — does NOT re-arm. Prefer hold's inverse: resume + arm.
   graph-setup
            print how to set up the code-graph indexes workers query instead of
@@ -525,23 +536,62 @@ try {
     }
 
     case "restart": {
-      // Inherit the running daemon's port and project: a restart that quietly
-      // moved to the default port would leave every existing health check
-      // pointing at nothing. When the unit owns the live pid, restartDaemon
-      // goes through systemctl so the replacement stays supervised.
-      const { previous, record, via } = await restartDaemon({
-        port: portFlag(argv),
-        project: flag(argv, "project"),
-      });
-      if (previous !== undefined) {
+      if (argv.includes("--now")) {
+        // Inherit the running daemon's port and project: a restart that quietly
+        // moved to the default port would leave every existing health check
+        // pointing at nothing. When the unit owns the live pid, restartDaemon
+        // goes through systemctl so the replacement stays supervised.
+        const { previous, record, via } = await restartDaemon({
+          port: portFlag(argv),
+          project: flag(argv, "project"),
+        });
+        if (previous !== undefined) {
+          process.stdout.write(
+            `stopped — pid ${previous.pid}${via === "systemctl" ? " (via systemctl)" : ""}\n`,
+          );
+        }
+        process.stdout.write(
+          `started — pid ${record.pid}, /healthz on :${record.port}` +
+            `${record.project === undefined ? "" : `, project ${record.project}`}` +
+            `${via === "systemctl" ? " (via systemctl)" : ""}\nlog ${record.logFile}\n`,
+        );
+        break;
+      }
+
+      // Default: drain the fleet before restarting — pause new claims, wait for
+      // live workers to reach 0/N (bounded by --timeout), restart, then restore
+      // the prior dispatch state. A timed-out drain restarts nothing and leaves
+      // dispatch paused, so it is safe to run while the daemon is wedged.
+      const timeoutRaw = flag(argv, "timeout");
+      const timeoutSeconds =
+        timeoutRaw === undefined ? 1800 : Number.parseInt(timeoutRaw, 10);
+      const timeoutMs = (Number.isNaN(timeoutSeconds) ? 1800 : timeoutSeconds) * 1000;
+      let result!: RestartResult;
+      const deps: UpgradeDeps = {
+        ...DEFAULT_DEPS,
+        setPaused: (v) => setPaused(v, { source: "restart", reason: "restart, draining" }),
+        restartDaemon: async () => {
+          result = await restartDaemon({
+            port: portFlag(argv),
+            project: flag(argv, "project"),
+          });
+        },
+      };
+      try {
+        await drainAndRestart(deps, { project: flag(argv, "project"), timeoutMs });
+      } catch (err) {
+        process.stderr.write(`${err instanceof Error ? err.message : String(err)}\n`);
+        process.exit(1);
+      }
+      if (result.previous !== undefined) {
         process.stdout.write(
-          `stopped — pid ${previous.pid}${via === "systemctl" ? " (via systemctl)" : ""}\n`,
+          `stopped — pid ${result.previous.pid}${result.via === "systemctl" ? " (via systemctl)" : ""}\n`,
         );
       }
       process.stdout.write(
-        `started — pid ${record.pid}, /healthz on :${record.port}` +
-          `${record.project === undefined ? "" : `, project ${record.project}`}` +
-          `${via === "systemctl" ? " (via systemctl)" : ""}\nlog ${record.logFile}\n`,
+        `started — pid ${result.record.pid}, /healthz on :${result.record.port}` +
+          `${result.record.project === undefined ? "" : `, project ${result.record.project}`}` +
+          `${result.via === "systemctl" ? " (via systemctl)" : ""}\nlog ${result.record.logFile}\n`,
       );
       break;
     }
@@ -602,9 +652,19 @@ try {
       break;
     }
 
-    case "board":
-      await runBoard(flag(argv, "project"));
+    case "board": {
+      const project = flag(argv, "project");
+      // Headless one-shot mode: `--json` asks for it explicitly, and a
+      // non-interactive stdin or stdout (pipe, redirect, cron, script) cannot
+      // host the key-driven board anyway. The TTY throw inside runBoard stays
+      // as a backstop for direct callers.
+      if (argv.includes("--json") || !process.stdout.isTTY || !process.stdin.isTTY) {
+        process.stdout.write(`${boardJson(await boardSnapshotOnce(project))}\n`);
+        break;
+      }
+      await runBoard(project);
       break;
+    }
 
     case "hold": {
       const r = hold(flag(argv, "project"));
@@ -732,6 +792,7 @@ try {
       try {
         const outcome = await unblockIssue(project, makeTracker(project), store, issue, {
           force: argv.includes("--force"),
+          requeue: !argv.includes("--no-requeue"),
         });
         process.stdout.write(`${formatUnblock(issue, outcome, project, resolveCaps(project, cfg.defaults))}\n`);
         // A refusal must not read as success to a script or a board keypress.
@@ -1006,7 +1067,7 @@ try {
     }
 
     case "pause":
-      setPaused(true);
+      setPaused(true, { source: "pause", reason: flag(argv, "reason") });
       process.stdout.write(
         "paused — no new work will be claimed\n" +
           "note: ticks keep firing if armed; use hold to silence both\n",

package/src/config.ts

@@ -15,6 +15,7 @@
 import { chmodSync, existsSync, mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { dirname, isAbsolute, join } from "node:path";
+import { backupTimestamp, copyToUniqueBackup } from "./backups.ts";
 import {
   AUTHORITY_HOLDERS,
   BASE_FRESHNESS,
@@ -103,6 +104,11 @@ export function stateDir(): string {
   return dirname(configPath());
 }
 
+/** State-root directory for durable copies of every pre-write config.json. */
+export function configBackupDir(): string {
+  return join(stateDir(), "backups", "config");
+}
+
 /**
  * Where per-run worktrees and bare mirrors live when a project names neither.
  *
@@ -188,6 +194,13 @@ export function writeConfigRaw(text: string): void {
   if (created !== undefined) chmodSync(dir, 0o700);
 
   const target = configPath();
+  // A durable pre-write copy under state keeps the previous bytes recoverable
+  // even when the process dies between this moment and the rename, or when a
+  // later caller discovers it needs the old file back. Retention is unbounded,
+  // matching the brief backups.
+  if (existsSync(target)) {
+    copyToUniqueBackup(target, configBackupDir(), `config.json.bak-${backupTimestamp()}`);
+  }
   const tmp = join(dir, `.config.json.${process.pid.toString(36)}.${Date.now().toString(36)}.tmp`);
   try {
     writeFileSync(tmp, text, { mode: 0o600 });
@@ -208,6 +221,7 @@ export function resolveCaps(p: ProjectConfig, defaults: Caps): Caps {
   const o: Partial<Caps> = p.caps ?? {};
   return {
     maxConcurrentWorkers: o.maxConcurrentWorkers ?? defaults.maxConcurrentWorkers,
+    maxConcurrentWorkersPerRepo: o.maxConcurrentWorkersPerRepo ?? defaults.maxConcurrentWorkersPerRepo,
     dailySpendUsd: o.dailySpendUsd !== undefined ? o.dailySpendUsd : defaults.dailySpendUsd,
     planUsage: o.planUsage !== undefined ? o.planUsage : defaults.planUsage,
     workerMaxTurns: o.workerMaxTurns ?? defaults.workerMaxTurns,
@@ -408,6 +422,15 @@ function normalizeProject(
   if (nonEmptyString(rawQueueLabel)) queueLabel = rawQueueLabel;
   else problems.push(`${label}: queueLabel must be a non-empty string — it is the human sign-off gate`);
 
+  // Tolerant like the other optional fields: an integer >= 1 is a grooming
+  // trigger, and anything else degrades to absent (the tick default) rather
+  // than invalidating the whole project.
+  const rawGroomBelow = raw["groomBelow"];
+  const groomBelow =
+    typeof rawGroomBelow === "number" && Number.isInteger(rawGroomBelow) && rawGroomBelow >= 1
+      ? rawGroomBelow
+      : undefined;
+
   const routing = raw["routing"] as Raw | undefined;
   const rawPrefix = routing?.["labelPrefix"];
   const labelPrefix = typeof rawPrefix === "string" ? rawPrefix : DEFAULT_LABEL_PREFIX;
@@ -434,6 +457,7 @@ function normalizeProject(
     name,
     tracker: { kind: "github", repo: trackerRepo },
     queueLabel,
+    ...(groomBelow === undefined ? {} : { groomBelow }),
     stateLabels: {
       inProgress: pickString(stateLabels?.["inProgress"], DEFAULT_STATE_LABELS.inProgress),
       blocked: pickString(stateLabels?.["blocked"], DEFAULT_STATE_LABELS.blocked),