omp-conductor 0.15.13 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/REFERENCE.md +72 -2
  2. package/package.json +2 -1
  3. package/schema/config.schema.json +6 -0
  4. package/src/admission.ts +745 -0
  5. package/src/ask.ts +47 -0
  6. package/src/backups.ts +19 -7
  7. package/src/board.ts +1 -2
  8. package/src/briefs/orchestrator.md +62 -4
  9. package/src/cli.ts +26 -0
  10. package/src/commands/context.ts +3 -0
  11. package/src/commands/decision.ts +10 -1
  12. package/src/commands/doctor.ts +2 -0
  13. package/src/commands/message.ts +8 -1
  14. package/src/commands/restart.ts +15 -3
  15. package/src/commands/restore-db.ts +146 -0
  16. package/src/commands/stop.ts +24 -15
  17. package/src/commands/unfreeze.ts +56 -0
  18. package/src/commands/watch.ts +77 -0
  19. package/src/config-schema.ts +9 -0
  20. package/src/config.ts +24 -0
  21. package/src/daemon.ts +239 -530
  22. package/src/dashboard/server.ts +2 -1
  23. package/src/decisions.ts +32 -7
  24. package/src/depends-on.ts +73 -0
  25. package/src/doctor.ts +178 -5
  26. package/src/escalate.ts +114 -15
  27. package/src/failure-class.ts +47 -0
  28. package/src/fleet.ts +41 -410
  29. package/src/gitops.ts +86 -1
  30. package/src/log.ts +40 -0
  31. package/src/model-fallback.ts +3 -2
  32. package/src/omp-settings.ts +114 -0
  33. package/src/omp.ts +39 -0
  34. package/src/orchestrator-tick.ts +7 -1
  35. package/src/reports.ts +124 -12
  36. package/src/session-host.ts +6 -0
  37. package/src/setup-wizard.ts +36 -0
  38. package/src/setup.ts +58 -1
  39. package/src/status-render.ts +445 -0
  40. package/src/stop-provenance.ts +53 -0
  41. package/src/store.ts +352 -11
  42. package/src/types.ts +187 -4
  43. package/src/unblock.ts +1 -1
  44. package/src/upgrade-verify.ts +1 -1
  45. package/src/upgrade.ts +1 -2
  46. package/src/verbs/server.ts +25 -0
  47. package/src/worker.ts +162 -10
package/src/store.ts CHANGED
@@ -10,13 +10,24 @@
10
10
  */
11
11
 
12
12
  import { Database } from "bun:sqlite";
13
- import { mkdirSync } from "node:fs";
13
+ import { randomUUID } from "node:crypto";
14
+ import { existsSync, mkdirSync, rmSync } from "node:fs";
14
15
  import { dirname, join } from "node:path";
15
16
 
17
+ import { backupTimestamp, publishTempFile } from "./backups.ts";
16
18
  import { stateDir } from "./config.ts";
17
19
  import { DECISION_TTL_MS, DEFAULT_CAPS, DIGEST_BACKLOG_LIMIT } from "./types.ts";
20
+
21
+ /**
22
+ * `expiresAt` for a watch (#459). It has no human deadline — a condition that
23
+ * simply has not fired yet is a different thing from a question nobody
24
+ * answered, and silently expiring it could drop a release. Far enough ahead
25
+ * that `expireDueDecisions` (`expiresAt <= now`) never closes it.
26
+ */
27
+ const DECISION_WATCH_NEVER_EXPIRES = Number.MAX_SAFE_INTEGER;
18
28
  import { dispatchInfra, neverStarted } from "./failure-class.ts";
19
29
  import type {
30
+ BaseFreeze,
20
31
  BaseHealth,
21
32
  DecisionDraft,
22
33
  FailureClass,
@@ -121,6 +132,14 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
121
132
  turns: true,
122
133
  maxTurns: true,
123
134
  spendUsd: true,
135
+ provider429Count: true,
136
+ resolvedModel: true,
137
+ resolvedProvider: true,
138
+ retryFallbacks: true,
139
+ retryFallbackSucceeded: true,
140
+ modelRecoveries: true,
141
+ autoRetryCount: true,
142
+ autoCompactionCount: true,
124
143
  sessionFile: true,
125
144
  prUrl: true,
126
145
  headSha: true,
@@ -158,6 +177,14 @@ interface RunRow {
158
177
  turns: number;
159
178
  maxTurns: number;
160
179
  spendUsd: number;
180
+ provider429Count: number | null;
181
+ resolvedModel: string | null;
182
+ resolvedProvider: string | null;
183
+ retryFallbacks: string | null;
184
+ retryFallbackSucceeded: number | null;
185
+ modelRecoveries: number | null;
186
+ autoRetryCount: number | null;
187
+ autoCompactionCount: number | null;
161
188
  sessionFile: string | null;
162
189
  prUrl: string | null;
163
190
  headSha: string | null;
@@ -204,6 +231,33 @@ function toBaseHealth(row: BaseHealthRow): BaseHealth {
204
231
  return health;
205
232
  }
206
233
 
234
+ /** The `base_freeze` table exactly as SQLite hands it back. */
235
+ interface BaseFreezeRow {
236
+ project: string;
237
+ repo: string;
238
+ culpritSha: string;
239
+ detail: string | null;
240
+ setAt: number;
241
+ clearedAt: number | null;
242
+ clearedBy: string | null;
243
+ clearedReason: string | null;
244
+ }
245
+
246
+ function toBaseFreeze(row: BaseFreezeRow): BaseFreeze {
247
+ const freeze: BaseFreeze = {
248
+ repo: row.repo,
249
+ culpritSha: row.culpritSha,
250
+ setAt: row.setAt,
251
+ };
252
+ if (row.detail !== null) freeze.detail = row.detail;
253
+ if (row.clearedAt !== null) {
254
+ freeze.clearedAt = row.clearedAt;
255
+ if (row.clearedBy !== null) freeze.clearedBy = row.clearedBy;
256
+ if (row.clearedReason !== null) freeze.clearedReason = row.clearedReason;
257
+ }
258
+ return freeze;
259
+ }
260
+
207
261
  interface FrictionRollupRow {
208
262
  project: string;
209
263
  day: string;
@@ -264,7 +318,10 @@ interface ReportRow {
264
318
  updatedAt: number;
265
319
  nextAttemptAt: number;
266
320
  messageId: number | null;
321
+ messageIds: string | null;
267
322
  deliveredAt: number | null;
323
+ sentParts: number;
324
+ sentPartsHash: string | null;
268
325
  lastError: string | null;
269
326
  }
270
327
 
@@ -404,6 +461,14 @@ CREATE TABLE IF NOT EXISTS runs (
404
461
  turns INTEGER NOT NULL,
405
462
  maxTurns INTEGER NOT NULL,
406
463
  spendUsd REAL NOT NULL,
464
+ provider429Count INTEGER,
465
+ resolvedModel TEXT,
466
+ resolvedProvider TEXT,
467
+ retryFallbacks TEXT,
468
+ retryFallbackSucceeded INTEGER,
469
+ modelRecoveries INTEGER,
470
+ autoRetryCount INTEGER,
471
+ autoCompactionCount INTEGER,
407
472
  sessionFile TEXT,
408
473
  prUrl TEXT,
409
474
  headSha TEXT,
@@ -439,6 +504,23 @@ CREATE TABLE IF NOT EXISTS base_health (
439
504
  PRIMARY KEY (project, repo)
440
505
  );
441
506
 
507
+ -- A per-repo merge freeze (#283). While clearedAt IS NULL the repo is frozen:
508
+ -- prMergeVerb refuses further merges to it with the base-red-freeze refusal. Set
509
+ -- when a watched merge (or live base observation) turns the base red; the clear
510
+ -- holds the mechanical green recovery and the operator's unfreeze verb, and acts
511
+ -- as the durable override/clear audit ledger.
512
+ CREATE TABLE IF NOT EXISTS base_freeze (
513
+ project TEXT NOT NULL,
514
+ repo TEXT NOT NULL,
515
+ culpritSha TEXT NOT NULL,
516
+ detail TEXT,
517
+ setAt INTEGER NOT NULL,
518
+ clearedAt INTEGER,
519
+ clearedBy TEXT,
520
+ clearedReason TEXT,
521
+ PRIMARY KEY (project, repo)
522
+ );
523
+
442
524
  CREATE TABLE IF NOT EXISTS notifications (
443
525
  "key" TEXT PRIMARY KEY,
444
526
  at INTEGER NOT NULL
@@ -516,7 +598,10 @@ CREATE TABLE IF NOT EXISTS reports (
516
598
  updatedAt INTEGER NOT NULL,
517
599
  nextAttemptAt INTEGER NOT NULL,
518
600
  messageId INTEGER,
601
+ messageIds TEXT,
519
602
  deliveredAt INTEGER,
603
+ sentParts INTEGER NOT NULL DEFAULT 0,
604
+ sentPartsHash TEXT,
520
605
  lastError TEXT
521
606
  );
522
607
  CREATE UNIQUE INDEX IF NOT EXISTS reports_dedupe
@@ -624,6 +709,7 @@ CREATE TABLE IF NOT EXISTS merge_locks (
624
709
  CREATE TABLE IF NOT EXISTS decisions (
625
710
  id TEXT PRIMARY KEY,
626
711
  project TEXT NOT NULL,
712
+ kind TEXT NOT NULL DEFAULT 'question',
627
713
  question TEXT NOT NULL,
628
714
  blocks TEXT,
629
715
  askedAt INTEGER NOT NULL,
@@ -756,6 +842,29 @@ function toSql(value: unknown): SqlValue {
756
842
  * row hand-edited or written by a future version drops its flags rather than
757
843
  * throwing and taking `status` down with it.
758
844
  */
845
+ /**
846
+ * Within-run fallback pairs out of their JSON column, or undefined for
847
+ * anything that is not an array of from/to string pairs. Same defensive
848
+ * posture as {@link toSettlementFlags}: a row hand-edited or written by a
849
+ * future version drops its fallbacks rather than throwing.
850
+ */
851
+ function toRetryFallbacks(text: string): { from: string; to: string }[] | undefined {
852
+ try {
853
+ const value: unknown = JSON.parse(text);
854
+ if (!Array.isArray(value) || value.length === 0) return undefined;
855
+ const pairs = value.filter(
856
+ (pair): pair is { from: string; to: string } =>
857
+ typeof pair === "object" &&
858
+ pair !== null &&
859
+ typeof (pair as { from?: unknown }).from === "string" &&
860
+ typeof (pair as { to?: unknown }).to === "string",
861
+ );
862
+ return pairs.length === 0 ? undefined : pairs;
863
+ } catch {
864
+ return undefined;
865
+ }
866
+ }
867
+
759
868
  function toSettlementFlags(text: string): SettlementFlag[] | undefined {
760
869
  try {
761
870
  const value: unknown = JSON.parse(text);
@@ -810,6 +919,17 @@ function toRecord(row: RunRow): RunRecord {
810
919
  if (flags !== undefined) record.settlementFlags = flags;
811
920
  }
812
921
  if (row.report !== null) record.report = row.report;
922
+ if (row.provider429Count !== null) record.provider429Count = row.provider429Count;
923
+ if (row.resolvedModel !== null) record.resolvedModel = row.resolvedModel;
924
+ if (row.resolvedProvider !== null) record.resolvedProvider = row.resolvedProvider;
925
+ if (row.retryFallbacks !== null) {
926
+ const fallbacks = toRetryFallbacks(row.retryFallbacks);
927
+ if (fallbacks !== undefined) record.retryFallbacks = fallbacks;
928
+ }
929
+ if (row.retryFallbackSucceeded !== null) record.retryFallbackSucceeded = row.retryFallbackSucceeded;
930
+ if (row.modelRecoveries !== null) record.modelRecoveries = row.modelRecoveries;
931
+ if (row.autoRetryCount !== null) record.autoRetryCount = row.autoRetryCount;
932
+ if (row.autoCompactionCount !== null) record.autoCompactionCount = row.autoCompactionCount;
813
933
  if (row.failureClass !== null) record.failureClass = row.failureClass as FailureClass;
814
934
  if (row.recoveryAction !== null) record.recoveryAction = row.recoveryAction as RecoveryAction;
815
935
  if (row.recoveredAt !== null) record.recoveredAt = row.recoveredAt;
@@ -837,6 +957,18 @@ function toReport(row: ReportRow): ReportRecord {
837
957
  if (row.attemptId !== null) record.attemptId = row.attemptId;
838
958
  if (row.dedupeKey !== null) record.dedupeKey = row.dedupeKey;
839
959
  if (row.messageId !== null) record.messageId = row.messageId;
960
+ if (row.messageIds !== null) {
961
+ // Written as JSON by this store; a row that fails to parse reads as if
962
+ // no ids were recorded, which is the honest reading of unreadable bytes.
963
+ try {
964
+ const ids: unknown = JSON.parse(row.messageIds);
965
+ if (Array.isArray(ids) && ids.every((id) => typeof id === "number")) record.messageIds = ids;
966
+ } catch {
967
+ // treated as absent
968
+ }
969
+ }
970
+ if (row.sentParts !== 0) record.sentParts = row.sentParts;
971
+ if (row.sentPartsHash !== null) record.sentPartsHash = row.sentPartsHash;
840
972
  if (row.deliveredAt !== null) record.deliveredAt = row.deliveredAt;
841
973
  if (row.lastError !== null) record.lastError = row.lastError;
842
974
  return record;
@@ -858,6 +990,7 @@ function toMaterialEvent(row: MaterialEventRow): MaterialEvent {
858
990
  interface DecisionRow {
859
991
  id: string;
860
992
  project: string;
993
+ kind: string;
861
994
  question: string;
862
995
  blocks: string | null;
863
996
  askedAt: number;
@@ -874,6 +1007,10 @@ function toDecision(row: DecisionRow): DecisionRecord {
874
1007
  const record: DecisionRecord = {
875
1008
  id: row.id,
876
1009
  project: row.project,
1010
+ // Only `watch` is ever a watch: anything else — and especially the historic
1011
+ // NULL the migration backfilled as `question` — reads as a human question.
1012
+ // A real question may carry a condition; that must not relabel it.
1013
+ kind: row.kind === "watch" ? "watch" : "question",
877
1014
  question: row.question,
878
1015
  askedAt: row.askedAt,
879
1016
  expiresAt: row.expiresAt,
@@ -949,7 +1086,11 @@ function toDispatchSummary(text: string): DispatchSummary | undefined {
949
1086
  !Array.isArray(hold.issues) ||
950
1087
  hold.issues.length > 5 ||
951
1088
  hold.count < hold.issues.length ||
952
- !hold.issues.every((issue) => Number.isSafeInteger(issue) && issue > 0),
1089
+ !hold.issues.every((issue) => Number.isSafeInteger(issue) && issue > 0) ||
1090
+ (hold.details !== undefined &&
1091
+ (!Array.isArray(hold.details) ||
1092
+ hold.details.length > 5 ||
1093
+ !hold.details.every((detail) => typeof detail === "string" && detail.length > 0))),
953
1094
  )
954
1095
  ) {
955
1096
  return undefined;
@@ -1002,6 +1143,51 @@ export function dbPath(): string {
1002
1143
  return join(stateDir(), "conductor.db");
1003
1144
  }
1004
1145
 
1146
+ /** The stem every restorable `conductor.db` snapshot is published under. */
1147
+ export const DB_SNAPSHOT_STEM = "conductor.db.bak";
1148
+
1149
+ /**
1150
+ * A fresh, restorable copy of the store at `source`, published under `destDir`
1151
+ * with the same temp-then-atomic-publish convention as the config backups.
1152
+ *
1153
+ * Uses SQLite's own `VACUUM INTO`, never a plain file copy: the store opens
1154
+ * with `journal_mode=WAL`, and a `copyFileSync` can capture a torn database
1155
+ * while committed rows still sit uncheckpointed in the live WAL — a file that
1156
+ * exists, looks fresh and passes `integrity_check` yet is missing the newest
1157
+ * commits. `VACUUM INTO` reads the source as a whole (main file plus WAL) and
1158
+ * writes a single-file snapshot, so the result opens cleanly in a fresh
1159
+ * connection and contains every committed row.
1160
+ *
1161
+ * The snapshot runs against its own read of the source, deliberately not the
1162
+ * caller's writer connection, so a snapshot can be taken while a writer holds
1163
+ * an uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
1164
+ * refused.
1165
+ *
1166
+ * Returns the published snapshot path.
1167
+ */
1168
+ export function snapshotDb(source: string, destDir: string): string {
1169
+ if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
1170
+ if (!existsSync(source)) {
1171
+ throw new Error(`cannot snapshot ${source}: the store does not exist`);
1172
+ }
1173
+ mkdirSync(destDir, { recursive: true });
1174
+ const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
1175
+ let connection: Database | undefined;
1176
+ try {
1177
+ connection = new Database(source);
1178
+ connection.exec("PRAGMA busy_timeout = 5000;");
1179
+ // `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
1180
+ connection.exec(`VACUUM INTO '${temporary.replaceAll("'", "''")}'`);
1181
+ } catch (err) {
1182
+ // Never leave a half-written snapshot on disk.
1183
+ rmSync(temporary, { force: true });
1184
+ throw err;
1185
+ } finally {
1186
+ connection?.close();
1187
+ }
1188
+ return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp()}`);
1189
+ }
1190
+
1005
1191
  /** The UTC calendar day (`YYYY-MM-DD`) a moment falls on — the partition key
1006
1192
  * for the daemon's observed github call counter (#198). */
1007
1193
  export function utcDay(now: number = Date.now()): string {
@@ -1120,6 +1306,30 @@ export function openStore(dbPath: string): Store {
1120
1306
  if (!columns.some((column) => column.name === "model")) {
1121
1307
  db.exec("ALTER TABLE runs ADD COLUMN model TEXT");
1122
1308
  }
1309
+ // The count of in-session provider 429s a run recorded was first written by
1310
+ // the worker (#573). Rows predating the column are NULL, which is the honest
1311
+ // reading: the count was not recorded, so the classifier must not treat the
1312
+ // absence as "no rate limiting happened".
1313
+ if (!columns.some((column) => column.name === "provider429Count")) {
1314
+ db.exec("ALTER TABLE runs ADD COLUMN provider429Count INTEGER");
1315
+ }
1316
+ // The within-run harness reliability surface (#539/#581): the resolved
1317
+ // model/provider and the fallback/retry/compaction counters. Rows predating
1318
+ // the columns are NULL, the honest reading of "the collection side had not
1319
+ // landed yet" — never "the run was healthy".
1320
+ for (const [name, type] of [
1321
+ ["resolvedModel", "TEXT"],
1322
+ ["resolvedProvider", "TEXT"],
1323
+ ["retryFallbacks", "TEXT"],
1324
+ ["retryFallbackSucceeded", "INTEGER"],
1325
+ ["modelRecoveries", "INTEGER"],
1326
+ ["autoRetryCount", "INTEGER"],
1327
+ ["autoCompactionCount", "INTEGER"],
1328
+ ] as const) {
1329
+ if (!columns.some((column) => column.name === name)) {
1330
+ db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
1331
+ }
1332
+ }
1123
1333
  // Existing held-notice rows predate exact digest ownership (#274). A NULL
1124
1334
  // report id is truthful for already-digested history and means "still owed"
1125
1335
  // only while digestedAt is also NULL.
@@ -1138,6 +1348,30 @@ export function openStore(dbPath: string): Store {
1138
1348
  if (!heldNoticeColumns.some((column) => column.name === "urgent")) {
1139
1349
  db.exec("ALTER TABLE held_notices ADD COLUMN urgent INTEGER NOT NULL DEFAULT 0");
1140
1350
  }
1351
+ // Multi-part delivery (#566): reports created before splitting existed carry
1352
+ // a single `messageId` and no per-part state, which is the honest reading of
1353
+ // a row that predates the watermark. `sentParts` defaults to 0 — "nothing
1354
+ // confirmed" — so a retry of a historical row starts from the first part.
1355
+ const reportColumns = db.query<{ name: string }, []>("PRAGMA table_info(reports)").all();
1356
+ if (!reportColumns.some((column) => column.name === "messageIds")) {
1357
+ db.exec("ALTER TABLE reports ADD COLUMN messageIds TEXT");
1358
+ }
1359
+ if (!reportColumns.some((column) => column.name === "sentParts")) {
1360
+ db.exec("ALTER TABLE reports ADD COLUMN sentParts INTEGER NOT NULL DEFAULT 0");
1361
+ }
1362
+ if (!reportColumns.some((column) => column.name === "sentPartsHash")) {
1363
+ db.exec("ALTER TABLE reports ADD COLUMN sentPartsHash TEXT");
1364
+ }
1365
+ // The kind split (#459): a row is either a question a human must answer or a
1366
+ // watch the orchestrator set for itself. Databases written before the split
1367
+ // never marked it, and every such row was a question — `watch` has no
1368
+ // historical form, because the old `decision open --resolves-when` was the
1369
+ // orchestrator's own misjudgement, not a durable kind. Defaulting the
1370
+ // historical rows to `question` is the honest reading.
1371
+ const decisionColumns = db.query<{ name: string }, []>("PRAGMA table_info(decisions)").all();
1372
+ if (!decisionColumns.some((column) => column.name === "kind")) {
1373
+ db.exec("ALTER TABLE decisions ADD COLUMN kind TEXT NOT NULL DEFAULT 'question'");
1374
+ }
1141
1375
 
1142
1376
  // One-time repair, and the reason it lives here rather than in the classifier:
1143
1377
  // a turn-0 environment fault that was ALREADY classified `unknown` and
@@ -1323,6 +1557,43 @@ export function openStore(dbPath: string): Store {
1323
1557
  GROUP BY repo, baseRef
1324
1558
  ORDER BY repo ASC, baseRef ASC`,
1325
1559
  );
1560
+ // Re-open (or refresh) an active freeze. On conflict we reset the clear
1561
+ // columns — a red re-observation after an override must re-arm the freeze —
1562
+ // and take the newest culprit/evidence/setAt as the current truth.
1563
+ const setBaseFreezeRow = db.query<
1564
+ unknown,
1565
+ [string, string, string, SqlValue, number]
1566
+ >(
1567
+ `INSERT INTO base_freeze
1568
+ (project, repo, culpritSha, detail, setAt, clearedAt, clearedBy, clearedReason)
1569
+ VALUES (?, ?, ?, ?, ?, NULL, NULL, NULL)
1570
+ ON CONFLICT(project, repo) DO UPDATE SET
1571
+ culpritSha = excluded.culpritSha,
1572
+ detail = excluded.detail,
1573
+ setAt = excluded.setAt,
1574
+ clearedAt = NULL,
1575
+ clearedBy = NULL,
1576
+ clearedReason = NULL`,
1577
+ );
1578
+ const wasBaseFrozen = db.query<{ n: number }, [string, string]>(
1579
+ `SELECT COUNT(*) AS n FROM base_freeze
1580
+ WHERE project = ? AND repo = ? AND clearedAt IS NULL`,
1581
+ );
1582
+ const selectBaseFreeze = db.query<BaseFreezeRow, [string, string]>(
1583
+ `SELECT * FROM base_freeze WHERE project = ? AND repo = ?`,
1584
+ );
1585
+ const selectFreezes = db.query<BaseFreezeRow, [string]>(
1586
+ `SELECT * FROM base_freeze WHERE project = ?
1587
+ ORDER BY (clearedAt IS NULL) DESC, setAt DESC, repo ASC`,
1588
+ );
1589
+ const clearBaseFreezeRow = db.query<
1590
+ unknown,
1591
+ [number, string, string, string, string]
1592
+ >(
1593
+ `UPDATE base_freeze
1594
+ SET clearedAt = ?, clearedBy = ?, clearedReason = ?
1595
+ WHERE project = ? AND repo = ? AND clearedAt IS NULL`,
1596
+ );
1326
1597
  const countAttempts = db.query<{ n: number }, [string, number]>(
1327
1598
  `SELECT COUNT(*) AS n FROM runs WHERE project = ? AND issue = ?`,
1328
1599
  );
@@ -1335,14 +1606,14 @@ export function openStore(dbPath: string): Store {
1335
1606
  const countFailures = db.query<{ n: number }, [string, number]>(
1336
1607
  `SELECT COUNT(*) AS n FROM runs
1337
1608
  WHERE project = ? AND issue = ? AND state = 'failed'
1338
- AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'returned-for-revision'))`,
1609
+ AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity', 'returned-for-revision'))`,
1339
1610
  );
1340
1611
  const countContinuations = db.query<{ n: number }, [string, number]>(
1341
1612
  `SELECT COUNT(*) AS n FROM runs
1342
1613
  WHERE project = ? AND issue = ?
1343
1614
  AND (
1344
1615
  (state IN ('killed', 'orphaned', 'blocked')
1345
- AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient')))
1616
+ AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity')))
1346
1617
  OR (state = 'failed' AND failureClass = 'returned-for-revision')
1347
1618
  )`,
1348
1619
  );
@@ -1840,10 +2111,19 @@ export function openStore(dbPath: string): Store {
1840
2111
  );
1841
2112
  const deliverReportRow = db.query<unknown, SqlValue[]>(
1842
2113
  `UPDATE reports
1843
- SET state = 'delivered', attemptId = NULL, messageId = ?, deliveredAt = ?,
2114
+ SET state = 'delivered', attemptId = NULL, messageId = ?, messageIds = ?, deliveredAt = ?,
1844
2115
  updatedAt = ?, lastError = NULL
1845
2116
  WHERE id = ? AND state = 'sending' AND attemptId = ?`,
1846
2117
  );
2118
+ // Mid-split progress, written before the next part ships: the first
2119
+ // `sentParts` parts of the fingerprinted message are confirmed accepted, so
2120
+ // a crash or a definitive failure can only ever re-send the part that was in
2121
+ // flight (#566). Guarded by attempt id like every other transition.
2122
+ const markPartsSentRow = db.query<unknown, [number, string, number, string, string]>(
2123
+ `UPDATE reports
2124
+ SET sentParts = ?, sentPartsHash = ?, updatedAt = ?
2125
+ WHERE id = ? AND state = 'sending' AND attemptId = ?`,
2126
+ );
1847
2127
  const requeueReportRow = db.query<unknown, [number, string, number, string, string]>(
1848
2128
  `UPDATE reports
1849
2129
  SET state = 'pending', attemptId = NULL, nextAttemptAt = ?, lastError = ?, updatedAt = ?
@@ -1962,10 +2242,10 @@ export function openStore(dbPath: string): Store {
1962
2242
  },
1963
2243
  );
1964
2244
 
1965
- const insertDecision = db.query<never, [string, string, string, SqlValue, number, number, SqlValue, string]>(
2245
+ const insertDecision = db.query<never, [string, string, string, string, SqlValue, number, number, SqlValue, string]>(
1966
2246
  `INSERT INTO decisions
1967
- (id, project, question, blocks, askedAt, expiresAt, condition, state)
1968
- VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
2247
+ (id, project, kind, question, blocks, askedAt, expiresAt, condition, state)
2248
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1969
2249
  );
1970
2250
  const selectDecision = db.query<DecisionRow, [string]>(`SELECT * FROM decisions WHERE id = ?`);
1971
2251
  const selectOpenDecisions = db.query<DecisionRow, [string]>(
@@ -2276,6 +2556,41 @@ export function openStore(dbPath: string): Store {
2276
2556
  );
2277
2557
  },
2278
2558
 
2559
+ baseFreeze(project: string, repo: string): BaseFreeze | undefined {
2560
+ const row = selectBaseFreeze.get(project, repo);
2561
+ return row === null ? undefined : toBaseFreeze(row);
2562
+ },
2563
+
2564
+ freezes(project: string): BaseFreeze[] {
2565
+ return selectFreezes.all(project).map(toBaseFreeze);
2566
+ },
2567
+
2568
+ setBaseFreeze(
2569
+ project: string,
2570
+ freeze: { repo: string; culpritSha: string; detail?: string; setAt?: number },
2571
+ ): boolean {
2572
+ const setAt = freeze.setAt ?? Date.now();
2573
+ const wasActive = (wasBaseFrozen.get(project, freeze.repo)?.n ?? 0) > 0;
2574
+ setBaseFreezeRow.run(
2575
+ project,
2576
+ freeze.repo,
2577
+ freeze.culpritSha,
2578
+ freeze.detail ?? null,
2579
+ setAt,
2580
+ );
2581
+ return !wasActive;
2582
+ },
2583
+
2584
+ clearBaseFreeze(
2585
+ project: string,
2586
+ repo: string,
2587
+ by: string,
2588
+ reason: string,
2589
+ at = Date.now(),
2590
+ ): boolean {
2591
+ return clearBaseFreezeRow.run(at, by, reason, project, repo).changes > 0;
2592
+ },
2593
+
2279
2594
 
2280
2595
  salvagedRuns(project: string): RunRecord[] {
2281
2596
  return selectSalvaged.all(project).map(toRecord);
@@ -2660,14 +2975,33 @@ export function openStore(dbPath: string): Store {
2660
2975
  markReportDelivered(
2661
2976
  id: string,
2662
2977
  attemptId: string,
2663
- messageId: number | undefined,
2978
+ messageIds: readonly number[],
2664
2979
  at: number,
2665
2980
  ): boolean {
2981
+ // `messageId` stays the first accepted id so rows written before
2982
+ // multi-part sends and the operator-facing rendering never disagree.
2666
2983
  return (
2667
- deliverReportRow.run(toSql(messageId), at, at, id, attemptId).changes > 0
2984
+ deliverReportRow.run(
2985
+ toSql(messageIds[0]),
2986
+ toSql(messageIds.length === 0 ? null : JSON.stringify(messageIds)),
2987
+ at,
2988
+ at,
2989
+ id,
2990
+ attemptId,
2991
+ ).changes > 0
2668
2992
  );
2669
2993
  },
2670
2994
 
2995
+ markReportPartsSent(
2996
+ id: string,
2997
+ attemptId: string,
2998
+ sentParts: number,
2999
+ sentPartsHash: string,
3000
+ at: number,
3001
+ ): boolean {
3002
+ return markPartsSentRow.run(sentParts, sentPartsHash, at, id, attemptId).changes > 0;
3003
+ },
3004
+
2671
3005
  markReportUncertain(id: string, attemptId: string, error: string): boolean {
2672
3006
  return uncertainReportRow.run(error, id, attemptId).changes > 0;
2673
3007
  },
@@ -2743,12 +3077,18 @@ export function openStore(dbPath: string): Store {
2743
3077
  },
2744
3078
 
2745
3079
  createDecision(draft: DecisionDraft): DecisionRecord {
3080
+ const kind = draft.kind ?? "question";
2746
3081
  const record: DecisionRecord = {
2747
3082
  id: crypto.randomUUID(),
2748
3083
  project: draft.project,
3084
+ kind,
2749
3085
  question: draft.question,
2750
3086
  askedAt: draft.at,
2751
- expiresAt: draft.at + DECISION_TTL_MS,
3087
+ // The seven-day deadline is for a question a human never answered. A
3088
+ // watch has no such deadline: it is the orchestrator's own bookkeeping,
3089
+ // and expiring it silently could drop a release the day its condition
3090
+ // finally fires, so it is never due.
3091
+ expiresAt: kind === "watch" ? DECISION_WATCH_NEVER_EXPIRES : draft.at + DECISION_TTL_MS,
2752
3092
  state: "open",
2753
3093
  };
2754
3094
  if (draft.blocks !== undefined) record.blocks = draft.blocks;
@@ -2756,6 +3096,7 @@ export function openStore(dbPath: string): Store {
2756
3096
  insertDecision.run(
2757
3097
  record.id,
2758
3098
  record.project,
3099
+ record.kind,
2759
3100
  record.question,
2760
3101
  toSql(record.blocks),
2761
3102
  record.askedAt,