omp-conductor 0.15.12 → 0.16.0

package/src/commands/restore-db.ts

@@ -0,0 +1,146 @@
+/**
+ * `restore-db` — put `conductor.db` back to a restorable snapshot.
+ *
+ * The store is the verb ledger, the decision rows, the run rows and the
+ * material-event ledger (#579); a restore replaces the live file wholesale
+ * from a snapshot the snapshot primitive produced. It refuses to run while a
+ * daemon is live — overwriting the store a live dispatch loop is writing is a
+ * corruption path, and `livingDaemon()` is the existing detect for it — and
+ * replaces the file atomically (temp file in the same directory, then rename),
+ * then drops any stale `-wal`/`-shm` sidecars so the restored database opens
+ * cleanly. The pre-restore `conductor.db-wal`/`-shm` are gone by design: the
+ * old WAL belongs to the file being replaced.
+ */
+
+import { Database } from "bun:sqlite";
+import { randomUUID } from "node:crypto";
+import { copyFileSync, existsSync, readdirSync, renameSync, rmSync, statSync } from "node:fs";
+import { join, resolve } from "node:path";
+
+import type { CommandContext } from "./context.ts";
+import { dbBackupDirFor, loadConfig, stateDir } from "../config.ts";
+import { livingDaemon } from "../lifecycle.ts";
+import { DB_SNAPSHOT_STEM, dbPath } from "../store.ts";
+
+const RESTORE_DB_USAGE = `omp-conductor restore-db [SNAPSHOT]
+
+Put conductor.db back to a restorable snapshot, replacing the live file
+wholesale and dropping stale -wal/-shm sidecars so the result opens cleanly.
+
+Refuses to run while a daemon is live — restoring under a running daemon is a
+corruption path, not an interruption. Stop the daemon first
+(\`omp-conductor stop\`).
+
+usage:
+  omp-conductor restore-db               restore the newest snapshot in the
+                                         configured db backup directory
+  omp-conductor restore-db PATH          restore this exact snapshot file
+
+SNAPSHOT is a file the store snapshot primitive produced. A relative PATH is
+resolved against the working directory.
+
+Before restoring, the command verifies the restored database reads cleanly
+(PRAGMA integrity_check) and prints the verb_ledger / decisions / runs counts.`;
+
+/**
+ * The newest snapshot already published under `dir`, by mtime — the default
+ * restore source. Undefined when none exists (a fresh store with no snapshot).
+ */
+function newestSnapshot(dir: string): string | undefined {
+  let names: string[];
+  try {
+    names = readdirSync(dir).filter((name) => name.startsWith(DB_SNAPSHOT_STEM));
+  } catch {
+    return undefined;
+  }
+  if (names.length === 0) return undefined;
+  const withMtime = names
+    .map((name) => ({ name, mtime: statSync(join(dir, name)).mtimeMs }))
+    .sort((a, b) => b.mtime - a.mtime);
+  return join(dir, withMtime[0]!.name);
+}
+
+/** Replace `conductor.db` with `snapshot`, atomically, dropping stale sidecars. */
+function overwriteStoreFrom(snapshot: string): void {
+  const target = dbPath();
+  const temporary = join(stateDir(), `.${DB_SNAPSHOT_STEM}-restore.${process.pid}.${randomUUID()}.tmp`);
+  copyFileSync(snapshot, temporary);
+  renameSync(temporary, target);
+  rmSync(`${target}-wal`, { force: true });
+  rmSync(`${target}-shm`, { force: true });
+}
+
+interface RowsRead {
+  integrity: string;
+  runs: number;
+  decisions: number;
+  verbLedger: number;
+}
+
+/** Read the restored store back — the proof the restore landed cleanly. */
+function readRestoredStore(target: string): RowsRead {
+  const db = new Database(target);
+  try {
+    const integrity = (db.query<{ integrity_check: string }, []>("PRAGMA integrity_check").get() as {
+      integrity_check: string;
+    }).integrity_check;
+    const count = (table: string): number =>
+      (db.query<{ n: number }, []>(`SELECT COUNT(*) AS n FROM ${table}`).get() as { n: number }).n;
+    return { integrity, runs: count("runs"), decisions: count("decisions"), verbLedger: count("verb_ledger") };
+  } finally {
+    db.close();
+  }
+}
+
+export async function restoreDbCommand(ctx: CommandContext): Promise<void> {
+  // Help first: parsing stops before any config read or liveness check.
+  if (ctx.argv[1] === "--help" || ctx.argv[1] === "-h") {
+    process.stdout.write(`${RESTORE_DB_USAGE}\n`);
+    return;
+  }
+  // Host-scoped: the store is one host-wide file, so --project cannot narrow it.
+  if (ctx.projectFlag !== undefined) {
+    process.stderr.write(`omp-conductor: restore-db does not take a project — the store is host-wide\n`);
+    process.exit(2);
+  }
+  const positionals = ctx.argv.slice(1).filter((a) => !a.startsWith("--"));
+  const flags = ctx.argv.slice(1).filter((a) => a.startsWith("--"));
+  if (flags.length > 0) {
+    process.stderr.write(`omp-conductor: restore-db: unexpected argument "${flags[0]}"\n`);
+    process.exit(2);
+  }
+  if (positionals.length > 1) {
+    process.stderr.write(`omp-conductor: restore-db takes at most one snapshot path\n`);
+    process.exit(2);
+  }
+
+  // The corruption-path guard: never restore under a daemon that is writing.
+  const daemon = livingDaemon();
+  if (daemon !== undefined) {
+    throw new Error(
+      `restore-db refuses while a daemon is running (pid ${daemon.pid}) — it would overwrite the store a live dispatch is writing; stop it first (\`omp-conductor stop\`)`,
+    );
+  }
+
+  const cfg = loadConfig();
+  const snapDir = dbBackupDirFor(cfg);
+  const requested = positionals[0];
+  const snapshot = requested === undefined ? newestSnapshot(snapDir) : resolve(requested);
+  if (snapshot === undefined) {
+    throw new Error(
+      `no conductor.db snapshot in ${snapDir} — take one (the snapshot primitive) or pass an explicit snapshot path`,
+    );
+  }
+  if (!existsSync(snapshot)) {
+    throw new Error(`snapshot ${snapshot} does not exist`);
+  }
+
+  overwriteStoreFrom(snapshot);
+  const restored = readRestoredStore(dbPath());
+  const integrityLine =
+    restored.integrity === "ok" ? "integrity ok" : `integrity: ${restored.integrity}`;
+  process.stdout.write(
+    `restored conductor.db from ${snapshot}\n` +
+      `\u2003${integrityLine} · runs ${restored.runs} · decisions ${restored.decisions} · verb_ledger ${restored.verbLedger}\n`,
+  );
+}

package/src/commands/stop.ts

@@ -7,45 +7,77 @@
  */
 
 import type { CommandContext } from "./context.ts";
+import { buildStopProvenance, liveWorkload, refuseBusyDaemon } from "../stop-provenance.ts";
 import { hold, pinPaneHalt, stopConductorPane } from "../fleet.ts";
 import { stopDaemon } from "../lifecycle.ts";
+import { openStore, dbPath } from "../store.ts";
 
 export async function stopCommand(ctx: CommandContext): Promise<void> {
 const withPane = ctx.argv.includes("--pane");
-const targets = ctx.targetProjects().map((project) => {
-  // `stop` takes the fleet down, so it always disarms — `--keep-ticks` is a
-  // `hold` affordance. Assert the invariant instead of printing a path that
-  // might not exist.
-  const held = hold(project.name, "halt");
-  if (held.disarmed === undefined) throw new Error("stop must disarm ticks");
-  return {
-    project,
-    hold: { ...held, disarmed: held.disarmed },
-    pin: withPane ? pinPaneHalt(project.name).path : undefined,
-  };
-});
-const stop = await stopDaemon();
-const stopLine =
-  stop.kind === "not-running"
-    ? "daemon was not running"
-    : `daemon stopped — pid ${stop.pid}${stop.via === "systemctl" ? " (via systemctl)" : ""}`;
-for (const target of targets) {
-  if (target.pin !== undefined) {
-    const pane = await stopConductorPane(target.project.name);
-    process.stdout.write(
-      `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
-        `${stopLine}\n` +
-        `pane recovery pinned at ${target.pin}\n` +
-        `pane stop: ${pane.stopped} — ${pane.detail}\n` +
-        `  (conductor agent "${pane.agentName}" only — herdr-fleet.service was NOT stopped;\n` +
-        `   resume clears the pin when you want recovery again)\n`,
-    );
-  } else {
-    process.stdout.write(
-      `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
-        `${stopLine}\n` +
-        `pane left running (pass --pane to stop the conductor agent and pin recovery)\n`,
-    );
+const forced = ctx.argv.includes("--force");
+const store = openStore(dbPath());
+try {
+  // Refusal (#545): the shared daemon serves every configured project, so a
+  // stop orphans any project's live workers — not just the one named. Refuse
+  // while anything is live, naming project + issues, before any hold/disarm/
+  // pin side-effect, so a refused stop leaves the fleet untouched. `--force`
+  // skips the whole gate (so a wedged daemon stays stoppable even when the
+  // store is unreadable) and the override is recorded in the provenance.
+  if (!forced) refuseBusyDaemon(liveWorkload(store), { override: false, verb: "stop" });
+  const targets = ctx.targetProjects().map((project) => {
+    // `stop` takes the fleet down, so it always disarms — `--keep-ticks` is a
+    // `hold` affordance. Assert the invariant instead of printing a path that
+    // might not exist.
+    const held = hold(project.name, "halt");
+    if (held.disarmed === undefined) throw new Error("stop must disarm ticks");
+    return {
+      project,
+      hold: { ...held, disarmed: held.disarmed },
+      pin: withPane ? pinPaneHalt(project.name).path : undefined,
+    };
+  });
+  // Provenance (#378): the shared daemon serves every configured project, so
+  // the record names the request's own project (or the global scope) AND every
+  // sibling with its live-run count — no silent cross-project stop. A forced
+  // stop keeps the override visible in the control path, never silent.
+  const provenance = buildStopProvenance({
+    controlPath: forced ? "cli stop --force" : "cli stop",
+    reason: "operator stop",
+    scope: targets.length > 1 ? "global" : "project",
+    project: targets.length === 1 ? targets[0]!.project.name : undefined,
+  });
+  const stop = await stopDaemon({
+    provenance,
+    // The durable row is written by the lifecycle at the exact chokepoint —
+    // immediately before systemctl/SIGTERM, with the delivery method that is
+    // about to be used — which is what makes "provenance precedes signalling"
+    // provable rather than claimed.
+    record: (signal) => store.recordDaemonStop(signal.provenance),
+  });
+  const stopLine =
+    stop.kind === "not-running"
+      ? "daemon was not running"
+      : `daemon stopped — pid ${stop.pid}${stop.via === "systemctl" ? " (via systemctl)" : ""}`;
+  for (const target of targets) {
+    if (target.pin !== undefined) {
+      const pane = await stopConductorPane(target.project.name);
+      process.stdout.write(
+        `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
+          `${stopLine}\n` +
+          `pane recovery pinned at ${target.pin}\n` +
+          `pane stop: ${pane.stopped} — ${pane.detail}\n` +
+          `  (conductor agent "${pane.agentName}" only — herdr-fleet.service was NOT stopped;\n` +
+          `   resume clears the pin when you want recovery again)\n`,
+      );
+    } else {
+      process.stdout.write(
+        `stopped — claiming paused; ticks disarmed at ${target.hold.disarmed.path}\n` +
+          `${stopLine}\n` +
+          `pane left running (pass --pane to stop the conductor agent and pin recovery)\n`,
+      );
+    }
   }
+} finally {
+  store.close();
 }
 }

package/src/commands/unfreeze.ts

@@ -0,0 +1,56 @@
+/**
+ * `unfreeze <repo>` — the operator's sanctioned override for a base-red-freeze.
+ *
+ * A base-red-freeze is a mechanical gate: `prMergeVerb` refuses every
+ * `conductor_pr_merge` to a repo whose base is red, and the floor binds even
+ * the orchestrator to that refusal. This verb is the *one* sanctioned way to
+ * lift it early — after the operator has judged the base repaired (or the
+ * merge warranted anyway) — and it is ledger-recorded: the freeze row keeps
+ * `clearedBy`/`clearedReason`/`clearedAt` as its durable override audit, and a
+ * material event names the override for the queue digest. It is deliberately
+ * not label surgery or a raw DB edit, and it does not re-arm — a still-red base
+ * will re-freeze the repo on the next watch observation.
+ */
+
+import type { CommandContext } from "./context.ts";
+import { findProject, loadConfig } from "../config.ts";
+import { dbPath, openStore } from "../store.ts";
+
+export async function unfreezeCommand(ctx: CommandContext): Promise<void> {
+  const repo = ctx.argv[1];
+  if (repo === undefined || repo.length === 0 || repo.startsWith("-")) {
+    process.stderr.write("omp-conductor: unfreeze needs the routed repo name, e.g. `omp-conductor unfreeze api`\n");
+    process.exit(2);
+  }
+  const reason = ctx.flag("--reason") ?? "operator-override";
+  const cfg = loadConfig();
+  const project = findProject(cfg, ctx.projectFlag);
+  const store = openStore(dbPath());
+  try {
+    const now = Date.now();
+    const lifted = store.clearBaseFreeze(project.name, repo, "operator", reason, now);
+    if (!lifted) {
+      process.stdout.write(
+        `unfreeze: ${repo} has no active base-red freeze in ${project.name} (merges already allowed). Nothing to lift.\n`,
+      );
+      return;
+    }
+    store.recordMaterialEvent({
+      project: project.name,
+      category: "unfreeze",
+      summary: `operator unfroze merges to ${repo} (${reason})`,
+      evidence:
+        `An operator lifted the base-red-freeze on ${repo} with reason "${reason}". ` +
+        "This is the sanctioned override path — the freeze re-arms automatically if the base is " +
+        "observed red again.",
+      occurredAt: now,
+      recordedAt: now,
+    });
+    process.stdout.write(
+      `unfreeze: ${repo} base-red freeze lifted in ${project.name} (reason: ${reason}). ` +
+        "Merges to this repo resume; a still-red base will re-freeze on the next watch observation.\n",
+    );
+  } finally {
+    store.close();
+  }
+}

package/src/commands/watch.ts

@@ -0,0 +1,77 @@
+/**
+ * `watch` — set a condition for the orchestrator itself, with no human in the
+ * loop (#459).
+ *
+ * The split exists because the old `decision open --resolves-when` kept
+ * producing rows rendered as questions put to the operator — the fleet looked
+ * like it was waiting on a person when it was waiting on GitHub. A watch is a
+ * row the orchestrator opened for itself: either a condition the daemon checks
+ * (`--resolves-when`) that wakes the next tick when met, or a carry note the
+ * next tick should read, neither of which ever needs an operator answer. It is
+ * distinguished durably by its `kind`, never by whether it carries a
+ * condition — a real question may carry one too.
+ */
+
+import type { CommandContext } from "./context.ts";
+import { findProject, loadConfig } from "../config.ts";
+import { CONDITION_FORMS, parseCondition } from "../decisions.ts";
+import { dbPath, openStore } from "../store.ts";
+
+export async function watchCommand(ctx: CommandContext): Promise<void> {
+  const sub = ctx.argv[1];
+  const project = findProject(loadConfig(), ctx.projectFlag);
+  const store = openStore(dbPath());
+  try {
+    if (sub === "add") {
+      const note = ctx.flag("note")?.trim();
+      if (note === undefined || note.length === 0 || note.startsWith("--")) {
+        process.stderr.write("omp-conductor: watch add needs --note with what the next tick should know\n");
+        process.exit(2);
+      }
+      const condition = ctx.flag("resolves-when")?.trim();
+      if (condition !== undefined && parseCondition(condition) === undefined) {
+        process.stderr.write(
+          `omp-conductor: --resolves-when must be one of:\n${CONDITION_FORMS.map((f) => `  ${f}`).join("\n")}\n`,
+        );
+        process.exit(2);
+      }
+      const blocks = ctx.flag("blocks")?.trim();
+      const watch = store.createDecision({
+        project: project.name,
+        question: note,
+        kind: "watch",
+        ...(blocks === undefined || blocks.length === 0 ? {} : { blocks }),
+        ...(condition === undefined ? {} : { condition }),
+        at: Date.now(),
+      });
+      const wake = condition === undefined ? "read by the next tick" : "the daemon wakes the next tick when it is met";
+      process.stdout.write(`watch ${watch.id} added — ${wake} (no operator answer needed)\n`);
+      return;
+    }
+
+    if (sub === "list" || sub === undefined) {
+      const watches = store.openDecisions(project.name).filter((d) => d.kind === "watch");
+      if (watches.length === 0) {
+        process.stdout.write("no watches\n");
+        return;
+      }
+      const now = Date.now();
+      for (const d of watches) {
+        const condition =
+          d.condition === undefined ? "-" : d.conditionMetAt === undefined ? "pending" : "met";
+        const hours = Math.max(0, Math.round((now - d.askedAt) / 3_600_000));
+        process.stdout.write(
+          `${d.id}  ${hours}h  blocks:${d.blocks ?? "-"}  condition:${condition}  ${d.question}\n`,
+        );
+      }
+      return;
+    }
+
+    process.stderr.write(
+      `omp-conductor: unknown watch subcommand "${sub}" — expected add or list\n`,
+    );
+    process.exit(2);
+  } finally {
+    store.close();
+  }
+}

package/src/config-schema.ts

@@ -321,6 +321,10 @@ const projectSchema = z
     // usable, exactly like `workerModel`.
     modelFallbacks: z.unknown().optional(),
     modelFallbackThreshold: z.unknown().optional(),
+    // The fleet-owned omp settings overlay (#537): an opaque map omp's own
+    // schema owns. Conductor validates YAML shape only — the loader keeps it
+    // when it is a mapping and drops anything else, like `modelFallbacks`.
+    ompSettings: z.unknown().optional(),
     escalation: escalationSchema.optional(),
     authority: authoritySchema.optional(),
     releasePolicy: releasePolicySchema.optional(),
@@ -344,6 +348,11 @@ const configSchema = z
     $schema: z.string().optional().describe("Path to the shipped config.schema.json"),
     version: z.number().describe(`The config format version (${READABLE_CONFIG_VERSIONS.join(" or ")})`),
     defaults: capsSchema.optional(),
+    dbBackupDir: z
+      .string()
+      .min(1, "must name a directory")
+      .optional()
+      .describe("Absolute directory for restorable conductor.db snapshots; defaults to <stateDir>/backups/db"),
     projects: z.array(projectSchema).min(1, `"projects" must be a non-empty array — the dispatcher has nothing to service otherwise`),
   })
   .loose()

package/src/config.ts

@@ -124,6 +124,18 @@ export function configBackupDir(): string {
   return join(stateDir(), "backups", "config");
 }
 
+/**
+ * Where restorable `conductor.db` snapshots land: the configured `dbBackupDir`,
+ * or the state-root default `<stateDir()>/backups/db` beside the config
+ * backups when the field is absent. A `~/`-prefixed configured value is
+ * expanded like every other hand-written path field.
+ */
+export function dbBackupDirFor(cfg: Pick<ConductorConfig, "dbBackupDir"> | undefined): string {
+  const configured = cfg?.dbBackupDir;
+  const dir = configured !== undefined && configured.trim() !== "" ? expandHome(configured) : join(stateDir(), "backups", "db");
+  return dir;
+}
+
 /**
  * Where per-run worktrees and bare mirrors live when a project names neither.
  *
@@ -1099,6 +1111,17 @@ function finalizeProject(
       ? rawThreshold
       : undefined;
 
+  // The fleet-owned omp settings overlay (#537): an opaque map omp's own schema
+  // owns, so the loader validates YAML shape only — a non-mapping is dropped
+  // like an unusable `modelFallbacks` entry rather than failing the load, and
+  // an absent field keeps today's dispatch byte for byte. Everything inside the
+  // map is omp's to interpret, never conductor's.
+  const rawOmpSettings = p["ompSettings"];
+  const ompSettings =
+    typeof rawOmpSettings === "object" && rawOmpSettings !== null && !Array.isArray(rawOmpSettings)
+      ? (rawOmpSettings as Record<string, unknown>)
+      : undefined;
+
   // Marked critical-base/safety markers (commit SHAs or refs). A continuation
   // branch must contain every one before the dispatcher may reattach it;
   // unusable entries are dropped like `modelFallbacks` and an absent or empty
@@ -1128,6 +1151,7 @@ function finalizeProject(
     ...(workerModel === undefined ? {} : { workerModel }),
     ...(modelFallbacks.length === 0 ? {} : { modelFallbacks }),
     ...(modelFallbackThreshold === undefined ? {} : { modelFallbackThreshold }),
+    ...(ompSettings === undefined ? {} : { ompSettings }),
     escalation,
     authority,
     releasePolicy,