omp-conductor 0.15.12 → 0.16.0

package/src/store.ts

@@ -10,13 +10,24 @@
  */
 
 import { Database } from "bun:sqlite";
-import { mkdirSync } from "node:fs";
+import { randomUUID } from "node:crypto";
+import { existsSync, mkdirSync, rmSync } from "node:fs";
 import { dirname, join } from "node:path";
 
+import { backupTimestamp, publishTempFile } from "./backups.ts";
 import { stateDir } from "./config.ts";
 import { DECISION_TTL_MS, DEFAULT_CAPS, DIGEST_BACKLOG_LIMIT } from "./types.ts";
+
+/**
+ * `expiresAt` for a watch (#459). It has no human deadline — a condition that
+ * simply has not fired yet is a different thing from a question nobody
+ * answered, and silently expiring it could drop a release. Far enough ahead
+ * that `expireDueDecisions` (`expiresAt <= now`) never closes it.
+ */
+const DECISION_WATCH_NEVER_EXPIRES = Number.MAX_SAFE_INTEGER;
 import { dispatchInfra, neverStarted } from "./failure-class.ts";
 import type {
+  BaseFreeze,
   BaseHealth,
   DecisionDraft,
   FailureClass,
@@ -24,6 +35,8 @@ import type {
   DecisionState,
   DigestBacklog,
   DispatchSummary,
+  DaemonStop,
+  DaemonStopDraft,
   FrictionAdmissionReason,
   FrictionKind,
   FrictionObservation,
@@ -38,6 +51,9 @@ import type {
   MaterialEventDraft,
   LabelOp,
   MergeLock,
+  OrchestratorDownMode,
+  OrchestratorIncident,
+  OrchestratorIncidentDraft,
   ReportDeliveryState,
   ReportDraft,
   ReportEnqueue,
@@ -116,6 +132,14 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
   turns: true,
   maxTurns: true,
   spendUsd: true,
+  provider429Count: true,
+  resolvedModel: true,
+  resolvedProvider: true,
+  retryFallbacks: true,
+  retryFallbackSucceeded: true,
+  modelRecoveries: true,
+  autoRetryCount: true,
+  autoCompactionCount: true,
   sessionFile: true,
   prUrl: true,
   headSha: true,
@@ -153,6 +177,14 @@ interface RunRow {
   turns: number;
   maxTurns: number;
   spendUsd: number;
+  provider429Count: number | null;
+  resolvedModel: string | null;
+  resolvedProvider: string | null;
+  retryFallbacks: string | null;
+  retryFallbackSucceeded: number | null;
+  modelRecoveries: number | null;
+  autoRetryCount: number | null;
+  autoCompactionCount: number | null;
   sessionFile: string | null;
   prUrl: string | null;
   headSha: string | null;
@@ -199,6 +231,33 @@ function toBaseHealth(row: BaseHealthRow): BaseHealth {
   return health;
 }
 
+/** The `base_freeze` table exactly as SQLite hands it back. */
+interface BaseFreezeRow {
+  project: string;
+  repo: string;
+  culpritSha: string;
+  detail: string | null;
+  setAt: number;
+  clearedAt: number | null;
+  clearedBy: string | null;
+  clearedReason: string | null;
+}
+
+function toBaseFreeze(row: BaseFreezeRow): BaseFreeze {
+  const freeze: BaseFreeze = {
+    repo: row.repo,
+    culpritSha: row.culpritSha,
+    setAt: row.setAt,
+  };
+  if (row.detail !== null) freeze.detail = row.detail;
+  if (row.clearedAt !== null) {
+    freeze.clearedAt = row.clearedAt;
+    if (row.clearedBy !== null) freeze.clearedBy = row.clearedBy;
+    if (row.clearedReason !== null) freeze.clearedReason = row.clearedReason;
+  }
+  return freeze;
+}
+
 interface FrictionRollupRow {
   project: string;
   day: string;
@@ -259,7 +318,10 @@ interface ReportRow {
   updatedAt: number;
   nextAttemptAt: number;
   messageId: number | null;
+  messageIds: string | null;
   deliveredAt: number | null;
+  sentParts: number;
+  sentPartsHash: string | null;
   lastError: string | null;
 }
 
@@ -327,6 +389,65 @@ function toIntakeItem(row: IntakeRow): IntakeItem {
   return item;
 }
 
+/** The `orchestrator_incidents` table exactly as SQLite hands it back (#288). */
+interface OrchestratorIncidentRow {
+  project: string;
+  mode: string;
+  cause: string | null;
+  since: number;
+  diverted: number;
+}
+
+/** NULL columns become absent properties, matching the other row converters. */
+function toOrchestratorIncident(row: OrchestratorIncidentRow): OrchestratorIncident {
+  return {
+    project: row.project,
+    mode: row.mode as OrchestratorDownMode,
+    ...(row.cause === null ? {} : { cause: row.cause }),
+    since: row.since,
+    diverted: row.diverted,
+  };
+}
+
+/** The `daemon_stops` table exactly as SQLite hands it back (#378). */
+interface DaemonStopRow {
+  id: string;
+  at: number;
+  controlPath: string;
+  callerPid: number | null;
+  callerUid: number | null;
+  role: string | null;
+  scope: string;
+  project: string | null;
+  daemonPid: number | null;
+  runtimeDir: string | null;
+  affected: string;
+  reason: string;
+  unattributed: number;
+}
+
+/**
+ * NULL columns become absent properties, matching the other row converters:
+ * a record read back out of the store deep-equals the one that went in.
+ */
+function toDaemonStop(row: DaemonStopRow): DaemonStop {
+  return {
+    id: row.id,
+    at: row.at,
+    controlPath: row.controlPath,
+    ...(row.callerPid === null ? {} : { callerPid: row.callerPid }),
+    ...(row.callerUid === null ? {} : { callerUid: row.callerUid }),
+    ...(row.role === null ? {} : { role: row.role as SessionRole }),
+    scope: row.scope as "global" | "project",
+    ...(row.project === null ? {} : { project: row.project }),
+    ...(row.daemonPid === null ? {} : { daemonPid: row.daemonPid }),
+    ...(row.runtimeDir === null ? {} : { runtimeDir: row.runtimeDir }),
+    affected: JSON.parse(row.affected) as { project: string; live: number }[],
+    reason: row.reason,
+    unattributed: row.unattributed === 1,
+  };
+}
+
 const SCHEMA = `
 CREATE TABLE IF NOT EXISTS runs (
   id          TEXT    PRIMARY KEY,
@@ -340,6 +461,14 @@ CREATE TABLE IF NOT EXISTS runs (
   turns       INTEGER NOT NULL,
   maxTurns    INTEGER NOT NULL,
   spendUsd    REAL    NOT NULL,
+  provider429Count INTEGER,
+  resolvedModel TEXT,
+  resolvedProvider TEXT,
+  retryFallbacks TEXT,
+  retryFallbackSucceeded INTEGER,
+  modelRecoveries INTEGER,
+  autoRetryCount INTEGER,
+  autoCompactionCount INTEGER,
   sessionFile TEXT,
   prUrl       TEXT,
   headSha     TEXT,
@@ -375,6 +504,23 @@ CREATE TABLE IF NOT EXISTS base_health (
   PRIMARY KEY (project, repo)
 );
 
+-- A per-repo merge freeze (#283). While clearedAt IS NULL the repo is frozen:
+-- prMergeVerb refuses further merges to it with the base-red-freeze refusal. Set
+-- when a watched merge (or live base observation) turns the base red; the clear
+-- holds the mechanical green recovery and the operator's unfreeze verb, and acts
+-- as the durable override/clear audit ledger.
+CREATE TABLE IF NOT EXISTS base_freeze (
+  project      TEXT    NOT NULL,
+  repo         TEXT    NOT NULL,
+  culpritSha   TEXT    NOT NULL,
+  detail       TEXT,
+  setAt        INTEGER NOT NULL,
+  clearedAt    INTEGER,
+  clearedBy    TEXT,
+  clearedReason TEXT,
+  PRIMARY KEY (project, repo)
+);
+
 CREATE TABLE IF NOT EXISTS notifications (
   "key" TEXT    PRIMARY KEY,
   at    INTEGER NOT NULL
@@ -452,7 +598,10 @@ CREATE TABLE IF NOT EXISTS reports (
   updatedAt     INTEGER NOT NULL,
   nextAttemptAt INTEGER NOT NULL,
   messageId     INTEGER,
+  messageIds    TEXT,
   deliveredAt   INTEGER,
+  sentParts     INTEGER NOT NULL DEFAULT 0,
+  sentPartsHash TEXT,
   lastError     TEXT
 );
 CREATE UNIQUE INDEX IF NOT EXISTS reports_dedupe
@@ -560,6 +709,7 @@ CREATE TABLE IF NOT EXISTS merge_locks (
 CREATE TABLE IF NOT EXISTS decisions (
   id             TEXT    PRIMARY KEY,
   project        TEXT    NOT NULL,
+  kind           TEXT    NOT NULL DEFAULT 'question',
   question       TEXT    NOT NULL,
   blocks         TEXT,
   askedAt        INTEGER NOT NULL,
@@ -628,6 +778,44 @@ CREATE TABLE IF NOT EXISTS intake_items (
 );
 CREATE INDEX IF NOT EXISTS intake_items_project_state
   ON intake_items (project, state, createdAt);
+-- The embedded orchestrator's lost-liveness incident (#288): durable so a
+-- daemon restarted while its orchestrator is still down rediscovers the open
+-- incident instead of forgetting it, and the diverted counter accumulates
+-- across the outage and across restarts. One row per project; removed on
+-- recovery. since is the dedupe anchor for both the down page and its
+-- closing recovery page. The mode CHECK keeps the two causes distinct.
+CREATE TABLE IF NOT EXISTS orchestrator_incidents (
+  project TEXT    PRIMARY KEY,
+  mode    TEXT    NOT NULL CHECK (mode IN ('start-failed', 'crashed')),
+  cause   TEXT,
+  since   INTEGER NOT NULL,
+  diverted INTEGER NOT NULL DEFAULT 0
+);
+
+-- Every stop/restart of the shared daemon, and who asked for it and why
+-- (#378). Deliberately NOT partitioned by project: the daemon serves every
+-- configured project, so a record written by one project's CLI must be
+-- readable from another project's status the moment the daemon is down —
+-- that cross-project read is the whole point. Written before signalling on
+-- the mediated paths (cli stop/restart) and as an honest unattributed
+-- fallback when the daemon receives a signal with no mediated request.
+-- Explicit fields only: never raw argv, environment values or credentials.
+CREATE TABLE IF NOT EXISTS daemon_stops (
+  id           TEXT    PRIMARY KEY,
+  at           INTEGER NOT NULL,
+  controlPath  TEXT    NOT NULL,
+  callerPid    INTEGER,
+  callerUid    INTEGER,
+  role         TEXT,
+  scope        TEXT    NOT NULL,
+  project      TEXT,
+  daemonPid    INTEGER,
+  runtimeDir   TEXT,
+  affected     TEXT    NOT NULL,
+  reason       TEXT    NOT NULL,
+  unattributed INTEGER NOT NULL
+);
+CREATE INDEX IF NOT EXISTS daemon_stops_at ON daemon_stops (at);
 `;
 
 /**
@@ -654,6 +842,29 @@ function toSql(value: unknown): SqlValue {
  * row hand-edited or written by a future version drops its flags rather than
  * throwing and taking `status` down with it.
  */
+/**
+ * Within-run fallback pairs out of their JSON column, or undefined for
+ * anything that is not an array of from/to string pairs. Same defensive
+ * posture as {@link toSettlementFlags}: a row hand-edited or written by a
+ * future version drops its fallbacks rather than throwing.
+ */
+function toRetryFallbacks(text: string): { from: string; to: string }[] | undefined {
+  try {
+    const value: unknown = JSON.parse(text);
+    if (!Array.isArray(value) || value.length === 0) return undefined;
+    const pairs = value.filter(
+      (pair): pair is { from: string; to: string } =>
+        typeof pair === "object" &&
+        pair !== null &&
+        typeof (pair as { from?: unknown }).from === "string" &&
+        typeof (pair as { to?: unknown }).to === "string",
+    );
+    return pairs.length === 0 ? undefined : pairs;
+  } catch {
+    return undefined;
+  }
+}
+
 function toSettlementFlags(text: string): SettlementFlag[] | undefined {
   try {
     const value: unknown = JSON.parse(text);
@@ -708,6 +919,17 @@ function toRecord(row: RunRow): RunRecord {
     if (flags !== undefined) record.settlementFlags = flags;
   }
   if (row.report !== null) record.report = row.report;
+  if (row.provider429Count !== null) record.provider429Count = row.provider429Count;
+  if (row.resolvedModel !== null) record.resolvedModel = row.resolvedModel;
+  if (row.resolvedProvider !== null) record.resolvedProvider = row.resolvedProvider;
+  if (row.retryFallbacks !== null) {
+    const fallbacks = toRetryFallbacks(row.retryFallbacks);
+    if (fallbacks !== undefined) record.retryFallbacks = fallbacks;
+  }
+  if (row.retryFallbackSucceeded !== null) record.retryFallbackSucceeded = row.retryFallbackSucceeded;
+  if (row.modelRecoveries !== null) record.modelRecoveries = row.modelRecoveries;
+  if (row.autoRetryCount !== null) record.autoRetryCount = row.autoRetryCount;
+  if (row.autoCompactionCount !== null) record.autoCompactionCount = row.autoCompactionCount;
   if (row.failureClass !== null) record.failureClass = row.failureClass as FailureClass;
   if (row.recoveryAction !== null) record.recoveryAction = row.recoveryAction as RecoveryAction;
   if (row.recoveredAt !== null) record.recoveredAt = row.recoveredAt;
@@ -735,6 +957,18 @@ function toReport(row: ReportRow): ReportRecord {
   if (row.attemptId !== null) record.attemptId = row.attemptId;
   if (row.dedupeKey !== null) record.dedupeKey = row.dedupeKey;
   if (row.messageId !== null) record.messageId = row.messageId;
+  if (row.messageIds !== null) {
+    // Written as JSON by this store; a row that fails to parse reads as if
+    // no ids were recorded, which is the honest reading of unreadable bytes.
+    try {
+      const ids: unknown = JSON.parse(row.messageIds);
+      if (Array.isArray(ids) && ids.every((id) => typeof id === "number")) record.messageIds = ids;
+    } catch {
+      // treated as absent
+    }
+  }
+  if (row.sentParts !== 0) record.sentParts = row.sentParts;
+  if (row.sentPartsHash !== null) record.sentPartsHash = row.sentPartsHash;
   if (row.deliveredAt !== null) record.deliveredAt = row.deliveredAt;
   if (row.lastError !== null) record.lastError = row.lastError;
   return record;
@@ -756,6 +990,7 @@ function toMaterialEvent(row: MaterialEventRow): MaterialEvent {
 interface DecisionRow {
   id: string;
   project: string;
+  kind: string;
   question: string;
   blocks: string | null;
   askedAt: number;
@@ -772,6 +1007,10 @@ function toDecision(row: DecisionRow): DecisionRecord {
   const record: DecisionRecord = {
     id: row.id,
     project: row.project,
+    // Only `watch` is ever a watch: anything else — and especially the historic
+    // NULL the migration backfilled as `question` — reads as a human question.
+    // A real question may carry a condition; that must not relabel it.
+    kind: row.kind === "watch" ? "watch" : "question",
     question: row.question,
     askedAt: row.askedAt,
     expiresAt: row.expiresAt,
@@ -847,7 +1086,11 @@ function toDispatchSummary(text: string): DispatchSummary | undefined {
           !Array.isArray(hold.issues) ||
           hold.issues.length > 5 ||
           hold.count < hold.issues.length ||
-          !hold.issues.every((issue) => Number.isSafeInteger(issue) && issue > 0),
+          !hold.issues.every((issue) => Number.isSafeInteger(issue) && issue > 0) ||
+          (hold.details !== undefined &&
+            (!Array.isArray(hold.details) ||
+              hold.details.length > 5 ||
+              !hold.details.every((detail) => typeof detail === "string" && detail.length > 0))),
       )
     ) {
       return undefined;
@@ -900,6 +1143,51 @@ export function dbPath(): string {
   return join(stateDir(), "conductor.db");
 }
 
+/** The stem every restorable `conductor.db` snapshot is published under. */
+export const DB_SNAPSHOT_STEM = "conductor.db.bak";
+
+/**
+ * A fresh, restorable copy of the store at `source`, published under `destDir`
+ * with the same temp-then-atomic-publish convention as the config backups.
+ *
+ * Uses SQLite's own `VACUUM INTO`, never a plain file copy: the store opens
+ * with `journal_mode=WAL`, and a `copyFileSync` can capture a torn database
+ * while committed rows still sit uncheckpointed in the live WAL — a file that
+ * exists, looks fresh and passes `integrity_check` yet is missing the newest
+ * commits. `VACUUM INTO` reads the source as a whole (main file plus WAL) and
+ * writes a single-file snapshot, so the result opens cleanly in a fresh
+ * connection and contains every committed row.
+ *
+ * The snapshot runs against its own read of the source, deliberately not the
+ * caller's writer connection, so a snapshot can be taken while a writer holds
+ * an uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
+ * refused.
+ *
+ * Returns the published snapshot path.
+ */
+export function snapshotDb(source: string, destDir: string): string {
+  if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
+  if (!existsSync(source)) {
+    throw new Error(`cannot snapshot ${source}: the store does not exist`);
+  }
+  mkdirSync(destDir, { recursive: true });
+  const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
+  let connection: Database | undefined;
+  try {
+    connection = new Database(source);
+    connection.exec("PRAGMA busy_timeout = 5000;");
+    // `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
+    connection.exec(`VACUUM INTO '${temporary.replaceAll("'", "''")}'`);
+  } catch (err) {
+    // Never leave a half-written snapshot on disk.
+    rmSync(temporary, { force: true });
+    throw err;
+  } finally {
+    connection?.close();
+  }
+  return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp()}`);
+}
+
 /** The UTC calendar day (`YYYY-MM-DD`) a moment falls on — the partition key
  *  for the daemon's observed github call counter (#198). */
 export function utcDay(now: number = Date.now()): string {
@@ -1018,6 +1306,30 @@ export function openStore(dbPath: string): Store {
   if (!columns.some((column) => column.name === "model")) {
     db.exec("ALTER TABLE runs ADD COLUMN model TEXT");
   }
+  // The count of in-session provider 429s a run recorded was first written by
+  // the worker (#573). Rows predating the column are NULL, which is the honest
+  // reading: the count was not recorded, so the classifier must not treat the
+  // absence as "no rate limiting happened".
+  if (!columns.some((column) => column.name === "provider429Count")) {
+    db.exec("ALTER TABLE runs ADD COLUMN provider429Count INTEGER");
+  }
+  // The within-run harness reliability surface (#539/#581): the resolved
+  // model/provider and the fallback/retry/compaction counters. Rows predating
+  // the columns are NULL, the honest reading of "the collection side had not
+  // landed yet" — never "the run was healthy".
+  for (const [name, type] of [
+    ["resolvedModel", "TEXT"],
+    ["resolvedProvider", "TEXT"],
+    ["retryFallbacks", "TEXT"],
+    ["retryFallbackSucceeded", "INTEGER"],
+    ["modelRecoveries", "INTEGER"],
+    ["autoRetryCount", "INTEGER"],
+    ["autoCompactionCount", "INTEGER"],
+  ] as const) {
+    if (!columns.some((column) => column.name === name)) {
+      db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
+    }
+  }
   // Existing held-notice rows predate exact digest ownership (#274). A NULL
   // report id is truthful for already-digested history and means "still owed"
   // only while digestedAt is also NULL.
@@ -1036,6 +1348,30 @@ export function openStore(dbPath: string): Store {
   if (!heldNoticeColumns.some((column) => column.name === "urgent")) {
     db.exec("ALTER TABLE held_notices ADD COLUMN urgent INTEGER NOT NULL DEFAULT 0");
   }
+  // Multi-part delivery (#566): reports created before splitting existed carry
+  // a single `messageId` and no per-part state, which is the honest reading of
+  // a row that predates the watermark. `sentParts` defaults to 0 — "nothing
+  // confirmed" — so a retry of a historical row starts from the first part.
+  const reportColumns = db.query<{ name: string }, []>("PRAGMA table_info(reports)").all();
+  if (!reportColumns.some((column) => column.name === "messageIds")) {
+    db.exec("ALTER TABLE reports ADD COLUMN messageIds TEXT");
+  }
+  if (!reportColumns.some((column) => column.name === "sentParts")) {
+    db.exec("ALTER TABLE reports ADD COLUMN sentParts INTEGER NOT NULL DEFAULT 0");
+  }
+  if (!reportColumns.some((column) => column.name === "sentPartsHash")) {
+    db.exec("ALTER TABLE reports ADD COLUMN sentPartsHash TEXT");
+  }
+  // The kind split (#459): a row is either a question a human must answer or a
+  // watch the orchestrator set for itself. Databases written before the split
+  // never marked it, and every such row was a question — `watch` has no
+  // historical form, because the old `decision open --resolves-when` was the
+  // orchestrator's own misjudgement, not a durable kind. Defaulting the
+  // historical rows to `question` is the honest reading.
+  const decisionColumns = db.query<{ name: string }, []>("PRAGMA table_info(decisions)").all();
+  if (!decisionColumns.some((column) => column.name === "kind")) {
+    db.exec("ALTER TABLE decisions ADD COLUMN kind TEXT NOT NULL DEFAULT 'question'");
+  }
 
   // One-time repair, and the reason it lives here rather than in the classifier:
   // a turn-0 environment fault that was ALREADY classified `unknown` and
@@ -1221,6 +1557,43 @@ export function openStore(dbPath: string): Store {
       GROUP BY repo, baseRef
       ORDER BY repo ASC, baseRef ASC`,
   );
+  // Re-open (or refresh) an active freeze. On conflict we reset the clear
+  // columns — a red re-observation after an override must re-arm the freeze —
+  // and take the newest culprit/evidence/setAt as the current truth.
+  const setBaseFreezeRow = db.query<
+    unknown,
+    [string, string, string, SqlValue, number]
+  >(
+    `INSERT INTO base_freeze
+       (project, repo, culpritSha, detail, setAt, clearedAt, clearedBy, clearedReason)
+     VALUES (?, ?, ?, ?, ?, NULL, NULL, NULL)
+     ON CONFLICT(project, repo) DO UPDATE SET
+       culpritSha = excluded.culpritSha,
+       detail = excluded.detail,
+       setAt = excluded.setAt,
+       clearedAt = NULL,
+       clearedBy = NULL,
+       clearedReason = NULL`,
+  );
+  const wasBaseFrozen = db.query<{ n: number }, [string, string]>(
+    `SELECT COUNT(*) AS n FROM base_freeze
+      WHERE project = ? AND repo = ? AND clearedAt IS NULL`,
+  );
+  const selectBaseFreeze = db.query<BaseFreezeRow, [string, string]>(
+    `SELECT * FROM base_freeze WHERE project = ? AND repo = ?`,
+  );
+  const selectFreezes = db.query<BaseFreezeRow, [string]>(
+    `SELECT * FROM base_freeze WHERE project = ?
+      ORDER BY (clearedAt IS NULL) DESC, setAt DESC, repo ASC`,
+  );
+  const clearBaseFreezeRow = db.query<
+    unknown,
+    [number, string, string, string, string]
+  >(
+    `UPDATE base_freeze
+        SET clearedAt = ?, clearedBy = ?, clearedReason = ?
+      WHERE project = ? AND repo = ? AND clearedAt IS NULL`,
+  );
   const countAttempts = db.query<{ n: number }, [string, number]>(
     `SELECT COUNT(*) AS n FROM runs WHERE project = ? AND issue = ?`,
   );
@@ -1233,14 +1606,14 @@ export function openStore(dbPath: string): Store {
   const countFailures = db.query<{ n: number }, [string, number]>(
     `SELECT COUNT(*) AS n FROM runs
       WHERE project = ? AND issue = ? AND state = 'failed'
-        AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'returned-for-revision'))`,
+        AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity', 'returned-for-revision'))`,
   );
   const countContinuations = db.query<{ n: number }, [string, number]>(
     `SELECT COUNT(*) AS n FROM runs
       WHERE project = ? AND issue = ?
         AND (
           (state IN ('killed', 'orphaned', 'blocked')
-            AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient')))
+            AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity')))
           OR (state = 'failed' AND failureClass = 'returned-for-revision')
         )`,
   );
@@ -1365,6 +1738,23 @@ export function openStore(dbPath: string): Store {
   const insertNotified = db.query<unknown, [string, number]>(
     `INSERT OR IGNORE INTO notifications ("key", at) VALUES (?, ?)`,
   );
+  // Orchestrator-down incident (#288). One row per project; INSERT OR IGNORE is
+  // the dedupe so a flapping orchestrator opens (and pages) once per incident.
+  const selectOrchestratorIncident = db.query<OrchestratorIncidentRow, [string]>(
+    `SELECT project, mode, cause, since, diverted FROM orchestrator_incidents WHERE project = ?`,
+  );
+  const insertOrchestratorIncident = db.query<unknown, [string, string, string | null, number]>(
+    `INSERT OR IGNORE INTO orchestrator_incidents (project, mode, cause, since)
+     VALUES (?, ?, ?, ?)`,
+  );
+  const bumpOrchestratorIncident = db.query<unknown, [number, string]>(
+    `UPDATE orchestrator_incidents SET diverted = diverted + ? WHERE project = ?`,
+  );
+  const deleteOrchestratorIncident = db.query<OrchestratorIncidentRow, [string]>(
+    `DELETE FROM orchestrator_incidents
+      WHERE project = ?
+      RETURNING project, mode, cause, since, diverted`,
+  );
   const upsertDispatch = db.query<unknown, [string, string]>(
     `INSERT INTO dispatch_summaries (project, summary) VALUES (?, ?)
      ON CONFLICT(project) DO UPDATE SET summary = excluded.summary`,
@@ -1721,10 +2111,19 @@ export function openStore(dbPath: string): Store {
   );
   const deliverReportRow = db.query<unknown, SqlValue[]>(
     `UPDATE reports
-        SET state = 'delivered', attemptId = NULL, messageId = ?, deliveredAt = ?,
+        SET state = 'delivered', attemptId = NULL, messageId = ?, messageIds = ?, deliveredAt = ?,
             updatedAt = ?, lastError = NULL
       WHERE id = ? AND state = 'sending' AND attemptId = ?`,
   );
+  // Mid-split progress, written before the next part ships: the first
+  // `sentParts` parts of the fingerprinted message are confirmed accepted, so
+  // a crash or a definitive failure can only ever re-send the part that was in
+  // flight (#566). Guarded by attempt id like every other transition.
+  const markPartsSentRow = db.query<unknown, [number, string, number, string, string]>(
+    `UPDATE reports
+        SET sentParts = ?, sentPartsHash = ?, updatedAt = ?
+      WHERE id = ? AND state = 'sending' AND attemptId = ?`,
+  );
   const requeueReportRow = db.query<unknown, [number, string, number, string, string]>(
     `UPDATE reports
         SET state = 'pending', attemptId = NULL, nextAttemptAt = ?, lastError = ?, updatedAt = ?
@@ -1843,10 +2242,10 @@ export function openStore(dbPath: string): Store {
     },
   );
 
-  const insertDecision = db.query<never, [string, string, string, SqlValue, number, number, SqlValue, string]>(
+  const insertDecision = db.query<never, [string, string, string, string, SqlValue, number, number, SqlValue, string]>(
     `INSERT INTO decisions
-       (id, project, question, blocks, askedAt, expiresAt, condition, state)
-     VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+       (id, project, kind, question, blocks, askedAt, expiresAt, condition, state)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
   );
   const selectDecision = db.query<DecisionRow, [string]>(`SELECT * FROM decisions WHERE id = ?`);
   const selectOpenDecisions = db.query<DecisionRow, [string]>(
@@ -1889,6 +2288,16 @@ export function openStore(dbPath: string): Store {
       ORDER BY at DESC, rowid DESC
       LIMIT ?`,
   );
+  const insertDaemonStop = db.query<unknown, SqlValue[]>(
+    `INSERT INTO daemon_stops
+       (id, at, controlPath, callerPid, callerUid, role, scope, project, daemonPid, runtimeDir, affected, reason, unattributed)
+     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+  );
+  // `rowid` breaks the tie when two requests land in the same millisecond, so
+  // "newest first" never silently hides one stop behind another.
+  const selectLatestDaemonStop = db.query<DaemonStopRow, []>(
+    `SELECT * FROM daemon_stops ORDER BY at DESC, rowid DESC LIMIT 1`,
+  );
   // Breaking a stale claim and taking it must be one transaction, or two
   // daemons both see the stale row, both delete it, and both insert.
   const breakStaleMergeLock = db.query<unknown, [string, number]>(
@@ -2147,6 +2556,41 @@ export function openStore(dbPath: string): Store {
       );
     },
 
+    baseFreeze(project: string, repo: string): BaseFreeze | undefined {
+      const row = selectBaseFreeze.get(project, repo);
+      return row === null ? undefined : toBaseFreeze(row);
+    },
+
+    freezes(project: string): BaseFreeze[] {
+      return selectFreezes.all(project).map(toBaseFreeze);
+    },
+
+    setBaseFreeze(
+      project: string,
+      freeze: { repo: string; culpritSha: string; detail?: string; setAt?: number },
+    ): boolean {
+      const setAt = freeze.setAt ?? Date.now();
+      const wasActive = (wasBaseFrozen.get(project, freeze.repo)?.n ?? 0) > 0;
+      setBaseFreezeRow.run(
+        project,
+        freeze.repo,
+        freeze.culpritSha,
+        freeze.detail ?? null,
+        setAt,
+      );
+      return !wasActive;
+    },
+
+    clearBaseFreeze(
+      project: string,
+      repo: string,
+      by: string,
+      reason: string,
+      at = Date.now(),
+    ): boolean {
+      return clearBaseFreezeRow.run(at, by, reason, project, repo).changes > 0;
+    },
+
 
     salvagedRuns(project: string): RunRecord[] {
       return selectSalvaged.all(project).map(toRecord);
@@ -2223,6 +2667,58 @@ export function openStore(dbPath: string): Store {
       insertNotified.run(key, Date.now());
     },
 
+    orchestratorIncident(project: string): OrchestratorIncident | undefined {
+      const row = selectOrchestratorIncident.get(project);
+      return row === null ? undefined : toOrchestratorIncident(row);
+    },
+
+    openOrchestratorIncident(draft: OrchestratorIncidentDraft): boolean {
+      return (
+        insertOrchestratorIncident.run(
+          draft.project,
+          draft.mode,
+          draft.cause ?? null,
+          draft.since,
+        ).changes === 1
+      );
+    },
+
+    bumpOrchestratorDiverted(project: string, by = 1): void {
+      // Only meaningful while an incident is open: a healthy or external
+      // orchestrator has no row, and the UPDATE touches nothing.
+      bumpOrchestratorIncident.run(by, project);
+    },
+
+    closeOrchestratorIncident(project: string, _at: number): OrchestratorIncident | undefined {
+      const row = deleteOrchestratorIncident.get(project);
+      return row === null ? undefined : toOrchestratorIncident(row);
+    },
+
+    recordDaemonStop(draft: DaemonStopDraft): DaemonStop {
+      const entry: DaemonStop = { ...draft, id: crypto.randomUUID(), at: Date.now() };
+      insertDaemonStop.run(
+        entry.id,
+        entry.at,
+        entry.controlPath,
+        toSql(entry.callerPid),
+        toSql(entry.callerUid),
+        toSql(entry.role),
+        entry.scope,
+        toSql(entry.project),
+        toSql(entry.daemonPid),
+        toSql(entry.runtimeDir),
+        JSON.stringify(entry.affected),
+        entry.reason,
+        entry.unattributed ? 1 : 0,
+      );
+      return entry;
+    },
+
+    latestDaemonStop(): DaemonStop | undefined {
+      const row = selectLatestDaemonStop.get();
+      return row === null ? undefined : toDaemonStop(row);
+    },
+
     recordDispatch(project: string, summary: DispatchSummary): void {
       const previous = selectDispatch.get(project);
       upsertDispatch.run(project, JSON.stringify(summary));
@@ -2479,14 +2975,33 @@ export function openStore(dbPath: string): Store {
     markReportDelivered(
       id: string,
       attemptId: string,
-      messageId: number | undefined,
+      messageIds: readonly number[],
       at: number,
     ): boolean {
+      // `messageId` stays the first accepted id so rows written before
+      // multi-part sends and the operator-facing rendering never disagree.
       return (
-        deliverReportRow.run(toSql(messageId), at, at, id, attemptId).changes > 0
+        deliverReportRow.run(
+          toSql(messageIds[0]),
+          toSql(messageIds.length === 0 ? null : JSON.stringify(messageIds)),
+          at,
+          at,
+          id,
+          attemptId,
+        ).changes > 0
       );
     },
 
+    markReportPartsSent(
+      id: string,
+      attemptId: string,
+      sentParts: number,
+      sentPartsHash: string,
+      at: number,
+    ): boolean {
+      return markPartsSentRow.run(sentParts, sentPartsHash, at, id, attemptId).changes > 0;
+    },
+
     markReportUncertain(id: string, attemptId: string, error: string): boolean {
       return uncertainReportRow.run(error, id, attemptId).changes > 0;
     },
@@ -2562,12 +3077,18 @@ export function openStore(dbPath: string): Store {
     },
 
     createDecision(draft: DecisionDraft): DecisionRecord {
+      const kind = draft.kind ?? "question";
       const record: DecisionRecord = {
         id: crypto.randomUUID(),
         project: draft.project,
+        kind,
         question: draft.question,
         askedAt: draft.at,
-        expiresAt: draft.at + DECISION_TTL_MS,
+        // The seven-day deadline is for a question a human never answered. A
+        // watch has no such deadline: it is the orchestrator's own bookkeeping,
+        // and expiring it silently could drop a release the day its condition
+        // finally fires, so it is never due.
+        expiresAt: kind === "watch" ? DECISION_WATCH_NEVER_EXPIRES : draft.at + DECISION_TTL_MS,
         state: "open",
       };
       if (draft.blocks !== undefined) record.blocks = draft.blocks;
@@ -2575,6 +3096,7 @@ export function openStore(dbPath: string): Store {
       insertDecision.run(
         record.id,
         record.project,
+        record.kind,
         record.question,
         toSql(record.blocks),
         record.askedAt,