omp-conductor 0.15.11 → 0.15.13

package/src/setup.ts

@@ -155,6 +155,14 @@ export interface SetupAnswers {
    * survive an amend of some other area (#369).
    */
   groomBelow?: number;
+  /**
+   * Ordered fallback worker models and the provider-failure threshold that
+   * starts them. The wizard never asks for either — they are hand-edited —
+   * so they exist here only to survive an amend of some other area (#369,
+   * #286).
+   */
+  modelFallbacks?: string[];
+  modelFallbackThreshold?: number;
   telegramChatId?: string;
   /** Forum topic for tier-2 Telegram pages; absent keeps flat-chat 0.13 behaviour. */
   telegramTopicId?: number;
@@ -193,6 +201,13 @@ export interface SetupAnswers {
    * merge provenance gate.
    */
   recoveryMerges?: ProjectConfig["recoveryMerges"];
+  /**
+   * Hand-edited critical-base/safety markers (commit SHAs or refs) carried
+   * through setup unchanged, like {@link recoveryMerges}. The wizard never
+   * invents one; forgetting them during an unrelated amend would drop the
+   * stale-base interlock from a project that relies on it (#428).
+   */
+  criticalBase?: ProjectConfig["criticalBase"];
   /**
    * Whether to render `ORCHESTRATOR.md` into the project's workspace root. Not
    * part of the config — the brief is the operator's file, and the conductor
@@ -836,6 +851,10 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
       : {}),
     // A documented hand-edited key, so an unrelated amend must not delete it.
     ...(a.groomBelow === undefined ? {} : { groomBelow: a.groomBelow }),
+    // The failover chain is hand-edited too: dropping it on an unrelated amend
+    // would pin every run of the project onto a dead provider again (#286).
+    ...(a.modelFallbacks === undefined ? {} : { modelFallbacks: [...a.modelFallbacks] }),
+    ...(a.modelFallbackThreshold === undefined ? {} : { modelFallbackThreshold: a.modelFallbackThreshold }),
     escalation,
     authority: { ...a.authority },
     // Written out in full, never as the legacy string: the file then says which
@@ -847,6 +866,9 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
     ...(a.recoveryMerges === undefined
       ? {}
       : { recoveryMerges: a.recoveryMerges.map((entry) => ({ ...entry })) }),
+    // Hand-edited safety markers carried unchanged on an unrelated amend
+    // (#428): dropping them would silently disarm the stale-base interlock.
+    ...(a.criticalBase === undefined ? {} : { criticalBase: [...a.criticalBase] }),
     // Written out even when it is the default, so an operator amending the
     // volume has a line in the file to point at. Opting into availability makes
     // the schedule explicit and daily; omitting it preserves the preset's
@@ -1037,11 +1059,14 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
   // what keeps the rewritten config identical to the one that was read.
   if (p.workerModel !== undefined) answers.workerModel = p.workerModel;
   if (p.groomBelow !== undefined) answers.groomBelow = p.groomBelow;
+  if (p.modelFallbacks !== undefined) answers.modelFallbacks = [...p.modelFallbacks];
+  if (p.modelFallbackThreshold !== undefined) answers.modelFallbackThreshold = p.modelFallbackThreshold;
   if (p.escalation.telegramChatId !== undefined) answers.telegramChatId = p.escalation.telegramChatId;
   if (p.escalation.telegramTopicId !== undefined) answers.telegramTopicId = p.escalation.telegramTopicId;
   if (p.recoveryMerges !== undefined) {
     answers.recoveryMerges = p.recoveryMerges.map((entry) => ({ ...entry }));
   }
+  if (p.criticalBase !== undefined) answers.criticalBase = [...p.criticalBase];
   if (p.reporting?.digest.at !== undefined) answers.dailyDigestAt = p.reporting.digest.at;
   if (p.reporting?.digest.timezone !== undefined) {
     answers.reportingTimezone = p.reporting.digest.timezone;

package/src/stop-provenance.ts

@@ -0,0 +1,66 @@
+/**
+ * Provenance for stop/restart requests (#378): who asked, from which project,
+ * for which daemon, naming every affected project and its live-run count.
+ *
+ * Shared by the `stop` and `restart` verbs so a second convention cannot
+ * drift beside the first. Explicit facts only — caller pid/uid and session
+ * role, scope, control path, timestamp, reason — and specifically never raw
+ * argv, environment values or credentials: the incident this exists for was
+ * a stop nobody could attribute, and a "just serialise the command line"
+ * implementation would leak tokens into the audit trail.
+ */
+
+import { loadConfig } from "./config.ts";
+import { openStore, dbPath } from "./store.ts";
+import { SESSION_ROLE_ENV, type DaemonStopDraft } from "./types.ts";
+
+export interface StopProvenanceSpec {
+  /** The operator-visible control path: "cli stop", "cli restart", … */
+  controlPath: string;
+  /** Non-secret reason; a generated default is fine, never credentials. */
+  reason: string;
+  /** "global" when the request covered every project ("--all"); otherwise "project". */
+  scope: "global" | "project";
+  /** The originating project, when project-scoped. */
+  project?: string;
+}
+
+/**
+ * Build the provenance draft for one stop/restart request.
+ *
+ * `affected` is every configured project with its live-run count at request
+ * time, not just the targeted one: the shared daemon serves all of them, and
+ * a project-scoped stop still stops the siblings — the record names them
+ * instead of silently interrupting them.
+ *
+ * Pure: nothing is written. The command decides when to persist (at request
+ * entry for drain-style restarts whose signal may never arrive, and at the
+ * lifecycle chokepoint immediately before signalling for every mediated stop).
+ */
+export function buildStopProvenance(spec: StopProvenanceSpec): DaemonStopDraft {
+  const cfg = loadConfig();
+  const store = openStore(dbPath());
+  try {
+    const affected = cfg.projects.map((project) => ({
+      project: project.name,
+      live: store.liveRuns(project.name).length,
+    }));
+    return {
+      controlPath: spec.controlPath,
+      callerPid: process.pid,
+      ...(typeof process.getuid === "function" ? { callerUid: process.getuid() } : {}),
+      // The daemon stamps SESSION_ROLE_ENV on every session it spawns, so a
+      // stop that a worker session's tools trigger names itself as a worker;
+      // everything else — the operator's shell, a systemd unit, a script — is
+      // the orchestrator surface (same convention as the reporting CLI).
+      role: process.env[SESSION_ROLE_ENV] === "worker" ? "worker" : "orchestrator",
+      scope: spec.scope,
+      ...(spec.project === undefined ? {} : { project: spec.project }),
+      affected,
+      reason: spec.reason,
+      unattributed: false,
+    };
+  } finally {
+    store.close();
+  }
+}

package/src/store.ts

@@ -24,6 +24,8 @@ import type {
   DecisionState,
   DigestBacklog,
   DispatchSummary,
+  DaemonStop,
+  DaemonStopDraft,
   FrictionAdmissionReason,
   FrictionKind,
   FrictionObservation,
@@ -38,6 +40,9 @@ import type {
   MaterialEventDraft,
   LabelOp,
   MergeLock,
+  OrchestratorDownMode,
+  OrchestratorIncident,
+  OrchestratorIncidentDraft,
   ReportDeliveryState,
   ReportDraft,
   ReportEnqueue,
@@ -82,6 +87,7 @@ const ACTIVE_PLACEHOLDERS = ACTIVE_STATES.map(() => "?").join(", ");
 const FRICTION_HOLD_REASONS: ReadonlySet<FrictionAdmissionReason> = new Set([
   "failed-attempts",
   "continuations",
+  "stale-base",
   "parent-lookup-error",
   "issue-state-lookup-error",
   "open-pr-lookup-error",
@@ -133,6 +139,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
   failureClass: true,
   recoveryAction: true,
   recoveredAt: true,
+  model: true,
 };
 
 /** Everything SQLite will accept from us. */
@@ -169,6 +176,7 @@ interface RunRow {
   failureClass: string | null;
   recoveryAction: string | null;
   recoveredAt: number | null;
+  model: string | null;
 }
 
 /** The `base_health` table exactly as SQLite hands it back. */
@@ -324,6 +332,65 @@ function toIntakeItem(row: IntakeRow): IntakeItem {
   return item;
 }
 
+/** The `orchestrator_incidents` table exactly as SQLite hands it back (#288). */
+interface OrchestratorIncidentRow {
+  project: string;
+  mode: string;
+  cause: string | null;
+  since: number;
+  diverted: number;
+}
+
+/** NULL columns become absent properties, matching the other row converters. */
+function toOrchestratorIncident(row: OrchestratorIncidentRow): OrchestratorIncident {
+  return {
+    project: row.project,
+    mode: row.mode as OrchestratorDownMode,
+    ...(row.cause === null ? {} : { cause: row.cause }),
+    since: row.since,
+    diverted: row.diverted,
+  };
+}
+
+/** The `daemon_stops` table exactly as SQLite hands it back (#378). */
+interface DaemonStopRow {
+  id: string;
+  at: number;
+  controlPath: string;
+  callerPid: number | null;
+  callerUid: number | null;
+  role: string | null;
+  scope: string;
+  project: string | null;
+  daemonPid: number | null;
+  runtimeDir: string | null;
+  affected: string;
+  reason: string;
+  unattributed: number;
+}
+
+/**
+ * NULL columns become absent properties, matching the other row converters:
+ * a record read back out of the store deep-equals the one that went in.
+ */
+function toDaemonStop(row: DaemonStopRow): DaemonStop {
+  return {
+    id: row.id,
+    at: row.at,
+    controlPath: row.controlPath,
+    ...(row.callerPid === null ? {} : { callerPid: row.callerPid }),
+    ...(row.callerUid === null ? {} : { callerUid: row.callerUid }),
+    ...(row.role === null ? {} : { role: row.role as SessionRole }),
+    scope: row.scope as "global" | "project",
+    ...(row.project === null ? {} : { project: row.project }),
+    ...(row.daemonPid === null ? {} : { daemonPid: row.daemonPid }),
+    ...(row.runtimeDir === null ? {} : { runtimeDir: row.runtimeDir }),
+    affected: JSON.parse(row.affected) as { project: string; live: number }[],
+    reason: row.reason,
+    unattributed: row.unattributed === 1,
+  };
+}
+
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS runs (
   id          TEXT    PRIMARY KEY,
@@ -354,7 +421,8 @@ CREATE TABLE IF NOT EXISTS runs (
   report      TEXT,
   failureClass TEXT,
   recoveryAction TEXT,
-  recoveredAt INTEGER
+  recoveredAt INTEGER,
+  model       TEXT
 );
 CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
 CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
@@ -624,6 +692,44 @@ CREATE TABLE IF NOT EXISTS intake_items (
 );
 CREATE INDEX IF NOT EXISTS intake_items_project_state
   ON intake_items (project, state, createdAt);
+-- The embedded orchestrator's lost-liveness incident (#288): durable so a
+-- daemon restarted while its orchestrator is still down rediscovers the open
+-- incident instead of forgetting it, and the diverted counter accumulates
+-- across the outage and across restarts. One row per project; removed on
+-- recovery. since is the dedupe anchor for both the down page and its
+-- closing recovery page. The mode CHECK keeps the two causes distinct.
+CREATE TABLE IF NOT EXISTS orchestrator_incidents (
+  project TEXT    PRIMARY KEY,
+  mode    TEXT    NOT NULL CHECK (mode IN ('start-failed', 'crashed')),
+  cause   TEXT,
+  since   INTEGER NOT NULL,
+  diverted INTEGER NOT NULL DEFAULT 0
+);
+
+-- Every stop/restart of the shared daemon, and who asked for it and why
+-- (#378). Deliberately NOT partitioned by project: the daemon serves every
+-- configured project, so a record written by one project's CLI must be
+-- readable from another project's status the moment the daemon is down —
+-- that cross-project read is the whole point. Written before signalling on
+-- the mediated paths (cli stop/restart) and as an honest unattributed
+-- fallback when the daemon receives a signal with no mediated request.
+-- Explicit fields only: never raw argv, environment values or credentials.
+CREATE TABLE IF NOT EXISTS daemon_stops (
+  id           TEXT    PRIMARY KEY,
+  at           INTEGER NOT NULL,
+  controlPath  TEXT    NOT NULL,
+  callerPid    INTEGER,
+  callerUid    INTEGER,
+  role         TEXT,
+  scope        TEXT    NOT NULL,
+  project      TEXT,
+  daemonPid    INTEGER,
+  runtimeDir   TEXT,
+  affected     TEXT    NOT NULL,
+  reason       TEXT    NOT NULL,
+  unattributed INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS daemon_stops_at ON daemon_stops (at);
 `;
 
 /**
@@ -707,6 +813,7 @@ function toRecord(row: RunRow): RunRecord {
   if (row.failureClass !== null) record.failureClass = row.failureClass as FailureClass;
   if (row.recoveryAction !== null) record.recoveryAction = row.recoveryAction as RecoveryAction;
   if (row.recoveredAt !== null) record.recoveredAt = row.recoveredAt;
+  if (row.model !== null) record.model = row.model;
   return record;
 }
 
@@ -1006,6 +1113,13 @@ export function openStore(dbPath: string): Store {
       db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
     }
   }
+  // The model a run dispatched on was first written by the modelFallbacks
+  // failover chain (#286). Rows predating the column are NULL, which is the
+  // honest reading: only a chain-configured project records a model, and the
+  // harness's own downgrade stays on the worker result, never in this column.
+  if (!columns.some((column) => column.name === "model")) {
+    db.exec("ALTER TABLE runs ADD COLUMN model TEXT");
+  }
   // Existing held-notice rows predate exact digest ownership (#274). A NULL
   // report id is truthful for already-digested history and means "still owed"
   // only while digestedAt is also NULL.
@@ -1353,6 +1467,23 @@ export function openStore(dbPath: string): Store {
   const insertNotified = db.query<unknown, [string, number]>(
     `INSERT OR IGNORE INTO notifications ("key", at) VALUES (?, ?)`,
   );
+  // Orchestrator-down incident (#288). One row per project; INSERT OR IGNORE is
+  // the dedupe so a flapping orchestrator opens (and pages) once per incident.
+  const selectOrchestratorIncident = db.query<OrchestratorIncidentRow, [string]>(
+    `SELECT project, mode, cause, since, diverted FROM orchestrator_incidents WHERE project = ?`,
+  );
+  const insertOrchestratorIncident = db.query<unknown, [string, string, string | null, number]>(
+    `INSERT OR IGNORE INTO orchestrator_incidents (project, mode, cause, since)
+     VALUES (?, ?, ?, ?)`,
+  );
+  const bumpOrchestratorIncident = db.query<unknown, [number, string]>(
+    `UPDATE orchestrator_incidents SET diverted = diverted + ? WHERE project = ?`,
+  );
+  const deleteOrchestratorIncident = db.query<OrchestratorIncidentRow, [string]>(
+    `DELETE FROM orchestrator_incidents
+      WHERE project = ?
+      RETURNING project, mode, cause, since, diverted`,
+  );
   const upsertDispatch = db.query<unknown, [string, string]>(
     `INSERT INTO dispatch_summaries (project, summary) VALUES (?, ?)
      ON CONFLICT(project) DO UPDATE SET summary = excluded.summary`,
@@ -1877,6 +2008,16 @@ export function openStore(dbPath: string): Store {
       ORDER BY at DESC, rowid DESC
       LIMIT ?`,
   );
+  const insertDaemonStop = db.query<unknown, SqlValue[]>(
+    `INSERT INTO daemon_stops
+       (id, at, controlPath, callerPid, callerUid, role, scope, project, daemonPid, runtimeDir, affected, reason, unattributed)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  );
+  // `rowid` breaks the tie when two requests land in the same millisecond, so
+  // "newest first" never silently hides one stop behind another.
+  const selectLatestDaemonStop = db.query<DaemonStopRow, []>(
+    `SELECT * FROM daemon_stops ORDER BY at DESC, rowid DESC LIMIT 1`,
+  );
   // Breaking a stale claim and taking it must be one transaction, or two
   // daemons both see the stale row, both delete it, and both insert.
   const breakStaleMergeLock = db.query<unknown, [string, number]>(
@@ -2211,6 +2352,58 @@ export function openStore(dbPath: string): Store {
       insertNotified.run(key, Date.now());
     },
 
+    orchestratorIncident(project: string): OrchestratorIncident | undefined {
+      const row = selectOrchestratorIncident.get(project);
+      return row === null ? undefined : toOrchestratorIncident(row);
+    },
+
+    openOrchestratorIncident(draft: OrchestratorIncidentDraft): boolean {
+      return (
+        insertOrchestratorIncident.run(
+          draft.project,
+          draft.mode,
+          draft.cause ?? null,
+          draft.since,
+        ).changes === 1
+      );
+    },
+
+    bumpOrchestratorDiverted(project: string, by = 1): void {
+      // Only meaningful while an incident is open: a healthy or external
+      // orchestrator has no row, and the UPDATE touches nothing.
+      bumpOrchestratorIncident.run(by, project);
+    },
+
+    closeOrchestratorIncident(project: string, _at: number): OrchestratorIncident | undefined {
+      const row = deleteOrchestratorIncident.get(project);
+      return row === null ? undefined : toOrchestratorIncident(row);
+    },
+
+    recordDaemonStop(draft: DaemonStopDraft): DaemonStop {
+      const entry: DaemonStop = { ...draft, id: crypto.randomUUID(), at: Date.now() };
+      insertDaemonStop.run(
+        entry.id,
+        entry.at,
+        entry.controlPath,
+        toSql(entry.callerPid),
+        toSql(entry.callerUid),
+        toSql(entry.role),
+        entry.scope,
+        toSql(entry.project),
+        toSql(entry.daemonPid),
+        toSql(entry.runtimeDir),
+        JSON.stringify(entry.affected),
+        entry.reason,
+        entry.unattributed ? 1 : 0,
+      );
+      return entry;
+    },
+
+    latestDaemonStop(): DaemonStop | undefined {
+      const row = selectLatestDaemonStop.get();
+      return row === null ? undefined : toDaemonStop(row);
+    },
+
     recordDispatch(project: string, summary: DispatchSummary): void {
       const previous = selectDispatch.get(project);
       upsertDispatch.run(project, JSON.stringify(summary));

package/src/tracker/github.ts

@@ -24,6 +24,7 @@
 import { credentialedEnv } from "../gitops.ts";
 import { parsePrDiff } from "../diff-flags.ts";
 import type {
+  IssueComment,
   IssueState,
   IssueSnapshot,
   MergedPrInfo,
@@ -760,6 +761,36 @@ export function readyIssuesFromRest(raw: string): ReadyIssue[] {
   return issues;
 }
 
+/**
+ * Parse the REST `/repos/{owner}/{repo}/issues/{n}/comments` body — already
+ * projected by jq to `[{author, body}]` — onto {@link IssueComment}.
+ *
+ * The REST endpoint answers chronological order (oldest first), so the array
+ * order is the order the brief renders. Nulls and unrecognizable payloads are
+ * tolerated the same way {@link readyIssuesFromRest} tolerates them: an empty
+ * answer means "no comments", and *only* a thrown read means "could not tell" —
+ * the caller distinguishes the two, because eliding them is the #517 failure.
+ */
+export function commentsFrom(raw: string): IssueComment[] {
+  let parsed: unknown;
+  try {
+    parsed = JSON.parse(raw) as unknown;
+  } catch {
+    return [];
+  }
+  if (!Array.isArray(parsed)) return [];
+  const comments: IssueComment[] = [];
+  for (const entry of parsed) {
+    if (entry === null || typeof entry !== "object") continue;
+    const row = entry as { readonly [key: string]: unknown };
+    const author = row["author"];
+    const body = row["body"];
+    if (typeof author !== "string" || typeof body !== "string") continue;
+    comments.push({ author, body });
+  }
+  return comments;
+}
+
 /** One cached page per request URL: the ETag GitHub answered, the raw JSON
  *  body it covered, and whether that page advertised a rel="next" successor.
  *  Module-level default so every makeTracker() in the process (board probe,
@@ -1103,6 +1134,22 @@ export function makeTracker(
       return pages.flatMap(readyIssuesFromRest);
     },
 
+    async listComments(issue: number): Promise<IssueComment[]> {
+      // REST, like `issueSnapshot` — deliberately not `gh issue view
+      // --comments`, whose renderer goes through the interactive pager and
+      // prints nothing on a non-TTY while still exiting 0 (the #517 failure).
+      // `gh api` output is never paged. One page: the brief truncates a huge
+      // thread with an explicit marker rather than paginating.
+      return commentsFrom(
+        await runGh([
+          "api",
+          `repos/${repo}/issues/${issue}/comments`,
+          "--jq",
+          "[.[] | {author: .user.login, body: .body}]",
+        ]),
+      );
+    },
+
     async addLabel(issue: number, label: string): Promise<void> {
       try {
         // REST POST is natively idempotent: re-adding a label the issue already