omp-conductor 0.17.0 → 0.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. package/REFERENCE.md +12 -8
  2. package/package.json +1 -1
  3. package/schema/config.schema.json +40 -1
  4. package/src/admission.ts +263 -44
  5. package/src/ask.ts +39 -3
  6. package/src/availability.ts +27 -1
  7. package/src/backups.ts +2 -2
  8. package/src/briefs/orchestrator.md +1 -0
  9. package/src/briefs/worker.md +38 -19
  10. package/src/command-help.ts +8 -1
  11. package/src/command-manifest.ts +5 -2
  12. package/src/commands/arm.ts +6 -3
  13. package/src/commands/message.ts +32 -4
  14. package/src/commands/watch.ts +62 -3
  15. package/src/config-schema.ts +53 -0
  16. package/src/config.ts +97 -1
  17. package/src/daemon.ts +1479 -1483
  18. package/src/decisions.ts +51 -6
  19. package/src/depends-on.ts +261 -1
  20. package/src/diff-flags.ts +350 -0
  21. package/src/digest-schedule.ts +37 -0
  22. package/src/doctor.ts +310 -22
  23. package/src/escalate.ts +560 -57
  24. package/src/failure-class.ts +71 -15
  25. package/src/fleet.ts +189 -34
  26. package/src/gitops.ts +103 -24
  27. package/src/graph-health.ts +20 -7
  28. package/src/graph.ts +313 -68
  29. package/src/lifecycle.ts +43 -7
  30. package/src/omp.ts +42 -0
  31. package/src/orchestrator-tick.ts +430 -162
  32. package/src/release-policy.ts +177 -5
  33. package/src/routing.ts +11 -3
  34. package/src/session-host.ts +16 -0
  35. package/src/settlement.ts +1728 -0
  36. package/src/setup-host.ts +193 -4
  37. package/src/setup-install.ts +91 -30
  38. package/src/setup-wizard.ts +1257 -78
  39. package/src/setup.ts +153 -6
  40. package/src/status-render.ts +36 -4
  41. package/src/store.ts +411 -17
  42. package/src/tracker/github.ts +607 -12
  43. package/src/types.ts +331 -5
  44. package/src/upgrade.ts +50 -19
  45. package/src/verbs/actions.ts +66 -18
  46. package/src/verbs/protocol.ts +45 -0
  47. package/src/verbs/server.ts +270 -13
  48. package/src/worker.ts +239 -6
  49. package/src/worktree.ts +115 -8
  50. package/systemd/omp-conductor-recover.sh +73 -0
  51. package/systemd/recover-unit-test.sh +61 -0
package/src/store.ts CHANGED
@@ -11,7 +11,7 @@
11
11
 
12
12
  import { Database } from "bun:sqlite";
13
13
  import { randomUUID } from "node:crypto";
14
- import { existsSync, mkdirSync, rmSync } from "node:fs";
14
+ import { existsSync, mkdirSync, readdirSync, rmSync, unlinkSync } from "node:fs";
15
15
  import { dirname, join } from "node:path";
16
16
 
17
17
  import { backupTimestamp, publishTempFile } from "./backups.ts";
@@ -41,6 +41,7 @@ import type {
41
41
  FrictionKind,
42
42
  FrictionObservation,
43
43
  FrictionSignal,
44
+ HistoricalInfraCandidate,
44
45
  HeldNotice,
45
46
  InterruptCategory,
46
47
  HeldNoticeDraft,
@@ -60,6 +61,9 @@ import type {
60
61
  ReportKind,
61
62
  ReportRecord,
62
63
  RecoveryAction,
64
+ ReviewReason,
65
+ ReviewRevisionOutcome,
66
+ ReviewRevisionRecord,
63
67
  RunPatch,
64
68
  RunRecord,
65
69
  RunState,
@@ -99,6 +103,7 @@ const FRICTION_HOLD_REASONS: ReadonlySet<FrictionAdmissionReason> = new Set([
99
103
  "failed-attempts",
100
104
  "continuations",
101
105
  "stale-base",
106
+ "critical-base-verify-error",
102
107
  "parent-lookup-error",
103
108
  "issue-state-lookup-error",
104
109
  "open-pr-lookup-error",
@@ -141,6 +146,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
141
146
  autoRetryCount: true,
142
147
  autoCompactionCount: true,
143
148
  sessionFile: true,
149
+ resumedFromRunId: true,
144
150
  prUrl: true,
145
151
  headSha: true,
146
152
  mergeSha: true,
@@ -150,6 +156,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
150
156
  salvageSha: true,
151
157
  salvageError: true,
152
158
  salvageAckAt: true,
159
+ quarantineDetail: true,
153
160
  startedAt: true,
154
161
  endedAt: true,
155
162
  lastError: true,
@@ -186,6 +193,7 @@ interface RunRow {
186
193
  autoRetryCount: number | null;
187
194
  autoCompactionCount: number | null;
188
195
  sessionFile: string | null;
196
+ resumedFromRunId: string | null;
189
197
  prUrl: string | null;
190
198
  headSha: string | null;
191
199
  mergeSha: string | null;
@@ -195,6 +203,7 @@ interface RunRow {
195
203
  salvageSha: string | null;
196
204
  salvageError: string | null;
197
205
  salvageAckAt: number | null;
206
+ quarantineDetail: string | null;
198
207
  startedAt: number;
199
208
  endedAt: number | null;
200
209
  lastError: string | null;
@@ -361,6 +370,48 @@ interface MergeLockRow {
361
370
  at: number;
362
371
  }
363
372
 
373
+ /** The `review_revisions` table exactly as SQLite hands it back (#677). */
374
+ interface ReviewRevisionRow {
375
+ id: string;
376
+ project: string;
377
+ runId: string;
378
+ issue: number;
379
+ prUrl: string;
380
+ headSha: string;
381
+ findings: string;
382
+ round: number;
383
+ reason: string;
384
+ sessionFile: string | null;
385
+ requestedAt: number;
386
+ dispatchedAt: number | null;
387
+ settledAt: number | null;
388
+ outcome: string | null;
389
+ }
390
+
391
+ /**
392
+ * NULL columns become absent properties, matching the other row converters:
393
+ * a record read back out of the store deep-equals the one that went in.
394
+ */
395
+ function toReviewRevision(row: ReviewRevisionRow): ReviewRevisionRecord {
396
+ const record: ReviewRevisionRecord = {
397
+ id: row.id,
398
+ project: row.project,
399
+ runId: row.runId,
400
+ issue: row.issue,
401
+ prUrl: row.prUrl,
402
+ headSha: row.headSha,
403
+ findings: row.findings,
404
+ round: row.round,
405
+ reason: row.reason as ReviewReason,
406
+ requestedAt: row.requestedAt,
407
+ };
408
+ if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
409
+ if (row.dispatchedAt !== null) record.dispatchedAt = row.dispatchedAt;
410
+ if (row.settledAt !== null) record.settledAt = row.settledAt;
411
+ if (row.outcome !== null) record.outcome = row.outcome as ReviewRevisionOutcome;
412
+ return record;
413
+ }
414
+
364
415
  /** The `intake_items` table exactly as SQLite hands it back (#299). */
365
416
  interface IntakeRow {
366
417
  id: string;
@@ -470,6 +521,7 @@ CREATE TABLE IF NOT EXISTS runs (
470
521
  autoRetryCount INTEGER,
471
522
  autoCompactionCount INTEGER,
472
523
  sessionFile TEXT,
524
+ resumedFromRunId TEXT,
473
525
  prUrl TEXT,
474
526
  headSha TEXT,
475
527
  mergeSha TEXT,
@@ -492,6 +544,22 @@ CREATE TABLE IF NOT EXISTS runs (
492
544
  CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
493
545
  CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
494
546
 
547
+ -- The historical infrastructure reconciliation's per-project review cursor
548
+ -- (#638). A bounded pass examines settled ci-deterministic rows newest-first
549
+ -- and persists the boundary of the last row it definitively decided, so older
550
+ -- repairable rows are reached across later daemon starts instead of being
551
+ -- starved by repeated rescans of the newest non-matches. classifierVersion is
552
+ -- the signature-list fingerprint at the time the cursor advanced; when the
553
+ -- classifier learns a new signature the stamp goes stale and the pass restarts
554
+ -- from the newest row rather than skipping past newly recognised evidence.
555
+ CREATE TABLE IF NOT EXISTS historical_infra_review (
556
+ project TEXT PRIMARY KEY,
557
+ cursorStartedAt INTEGER NOT NULL,
558
+ cursorRowid INTEGER NOT NULL,
559
+ classifierVersion TEXT NOT NULL,
560
+ updatedAt INTEGER NOT NULL
561
+ );
562
+
495
563
  CREATE TABLE IF NOT EXISTS base_health (
496
564
  project TEXT NOT NULL,
497
565
  repo TEXT NOT NULL,
@@ -693,6 +761,31 @@ CREATE TABLE IF NOT EXISTS merge_locks (
693
761
  at INTEGER NOT NULL
694
762
  );
695
763
 
764
+ -- One durable review-revision request (#677): an orchestrator returned a
765
+ -- green, run-owned pull request to its worker with blocking findings. Every
766
+ -- field the daemon needs to resume the exact session is persisted BEFORE the
767
+ -- worker is woken -- the findings, the exact reviewed head, the round number,
768
+ -- and the target run/session -- and the row doubles as the duplicate guard: a
769
+ -- revision whose row exists pending is one nobody may queue again.
770
+ CREATE TABLE IF NOT EXISTS review_revisions (
771
+ id TEXT PRIMARY KEY,
772
+ project TEXT NOT NULL,
773
+ runId TEXT NOT NULL,
774
+ issue INTEGER NOT NULL,
775
+ prUrl TEXT NOT NULL,
776
+ headSha TEXT NOT NULL,
777
+ findings TEXT NOT NULL,
778
+ round INTEGER NOT NULL,
779
+ reason TEXT NOT NULL,
780
+ sessionFile TEXT,
781
+ requestedAt INTEGER NOT NULL,
782
+ dispatchedAt INTEGER,
783
+ settledAt INTEGER,
784
+ outcome TEXT
785
+ );
786
+ CREATE INDEX IF NOT EXISTS review_revisions_pending
787
+ ON review_revisions (project, dispatchedAt, runId);
788
+
696
789
  -- Questions the orchestrator has put to its operator, and their answers (#136).
697
790
  --
698
791
  -- The reason this is a table and not the session's memory: a question lived only
@@ -903,6 +996,7 @@ function toRecord(row: RunRow): RunRecord {
903
996
  startedAt: row.startedAt,
904
997
  };
905
998
  if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
999
+ if (row.resumedFromRunId !== null) record.resumedFromRunId = row.resumedFromRunId;
906
1000
  if (row.prUrl !== null) record.prUrl = row.prUrl;
907
1001
  if (row.headSha !== null) record.headSha = row.headSha;
908
1002
  if (row.mergeSha !== null) record.mergeSha = row.mergeSha;
@@ -912,6 +1006,7 @@ function toRecord(row: RunRow): RunRecord {
912
1006
  if (row.salvageSha !== null) record.salvageSha = row.salvageSha;
913
1007
  if (row.salvageError !== null) record.salvageError = row.salvageError;
914
1008
  if (row.salvageAckAt !== null) record.salvageAckAt = row.salvageAckAt;
1009
+ if (row.quarantineDetail !== null) record.quarantineDetail = row.quarantineDetail;
915
1010
  if (row.endedAt !== null) record.endedAt = row.endedAt;
916
1011
  if (row.lastError !== null) record.lastError = row.lastError;
917
1012
  if (row.settlementFlags !== null) {
@@ -1143,12 +1238,59 @@ export function dbPath(): string {
1143
1238
  return join(stateDir(), "conductor.db");
1144
1239
  }
1145
1240
 
1241
+ /**
1242
+ * Live worker count across the run store, opening the database WITHOUT
1243
+ * creating or upgrading it. The setup apply's quiescence preflight must count
1244
+ * workers before the acknowledged barrier, and database preparation is itself
1245
+ * one of the mutations that barrier must precede — so the observation that
1246
+ * gates it cannot be the thing that creates or migrates the store (#618). The
1247
+ * on-disk database is opened read-only (never created, never "prepared"); a
1248
+ * missing file reports zero because no run has ever been persisted, which the
1249
+ * caller only trusts when no daemon is running either. An unreadable or
1250
+ * schema-broken file throws rather than reading as zero — fail closed, never
1251
+ * absence.
1252
+ */
1253
+ export function liveWorkersReadOnly(project?: string): number {
1254
+ const path = dbPath();
1255
+ if (!existsSync(path)) return 0;
1256
+ let db: Database | undefined;
1257
+ try {
1258
+ db = new Database(path, { create: false, readonly: true });
1259
+ db.exec("PRAGMA busy_timeout = 5000;");
1260
+ const placeholders = LIVE_STATES.map(() => "?").join(", ");
1261
+ // `undefined` is the host-global selector: every project's live rows count,
1262
+ // never only the rows of a project literally named "" — a host-global
1263
+ // observation that scoped to `project = ''` hid every real named row and
1264
+ // read zero beside a running fleet (#651 review #3).
1265
+ const row =
1266
+ project === undefined
1267
+ ? db
1268
+ .query<{ n: number }, SqlValue[]>(
1269
+ `SELECT COUNT(*) AS n FROM runs WHERE state IN (${placeholders})`,
1270
+ )
1271
+ .get(...LIVE_STATES)
1272
+ : db
1273
+ .query<{ n: number }, SqlValue[]>(
1274
+ `SELECT COUNT(*) AS n FROM runs WHERE project = ? AND state IN (${placeholders})`,
1275
+ )
1276
+ .get(project, ...LIVE_STATES);
1277
+ return row?.n ?? 0;
1278
+ } catch (err) {
1279
+ throw new Error(
1280
+ `cannot read ${path} to prove the fleet is quiescent: ${err instanceof Error ? err.message : String(err)}`,
1281
+ );
1282
+ } finally {
1283
+ db?.close();
1284
+ }
1285
+ }
1286
+
1146
1287
  /** The stem every restorable `conductor.db` snapshot is published under. */
1147
1288
  export const DB_SNAPSHOT_STEM = "conductor.db.bak";
1148
1289
 
1149
1290
  /**
1150
- * A fresh, restorable copy of the store at `source`, published under `destDir`
1151
- * with the same temp-then-atomic-publish convention as the config backups.
1291
+ * A single-file, transaction-consistent copy of the store at `source` into
1292
+ * `target` the shared core of {@link snapshotDb} and the setup apply's
1293
+ * rollback capture.
1152
1294
  *
1153
1295
  * Uses SQLite's own `VACUUM INTO`, never a plain file copy: the store opens
1154
1296
  * with `journal_mode=WAL`, and a `copyFileSync` can capture a torn database
@@ -1158,34 +1300,79 @@ export const DB_SNAPSHOT_STEM = "conductor.db.bak";
1158
1300
  * writes a single-file snapshot, so the result opens cleanly in a fresh
1159
1301
  * connection and contains every committed row.
1160
1302
  *
1161
- * The snapshot runs against its own read of the source, deliberately not the
1162
- * caller's writer connection, so a snapshot can be taken while a writer holds
1163
- * an uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
1303
+ * Runs against its own read of the source, deliberately not the caller's
1304
+ * writer connection, so a snapshot can be taken while a writer holds an
1305
+ * uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
1164
1306
  * refused.
1165
- *
1166
- * Returns the published snapshot path.
1167
1307
  */
1168
- export function snapshotDb(source: string, destDir: string): string {
1308
+ export function vacuumInto(source: string, target: string): void {
1169
1309
  if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
1170
1310
  if (!existsSync(source)) {
1171
1311
  throw new Error(`cannot snapshot ${source}: the store does not exist`);
1172
1312
  }
1173
- mkdirSync(destDir, { recursive: true });
1174
- const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
1175
1313
  let connection: Database | undefined;
1176
1314
  try {
1177
1315
  connection = new Database(source);
1178
1316
  connection.exec("PRAGMA busy_timeout = 5000;");
1179
1317
  // `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
1180
- connection.exec(`VACUUM INTO '${temporary.replaceAll("'", "''")}'`);
1318
+ connection.exec(`VACUUM INTO '${target.replaceAll("'", "''")}'`);
1319
+ } finally {
1320
+ connection?.close();
1321
+ }
1322
+ }
1323
+
1324
+ /**
1325
+ * A fresh, restorable copy of the store at `source`, published under `destDir`
1326
+ * with the same temp-then-atomic-publish convention as the config backups.
1327
+ *
1328
+ * The stem carries `at` (default now) so the filename's timestamp is the
1329
+ * snapshot's logical instant — the cadence passes its tick time, keeping the
1330
+ * name, the day marker and the retention ordering in one timeline.
1331
+ *
1332
+ * Returns the published snapshot path.
1333
+ */
1334
+ export function snapshotDb(source: string, destDir: string, at: number = Date.now()): string {
1335
+ if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
1336
+ if (!existsSync(source)) {
1337
+ throw new Error(`cannot snapshot ${source}: the store does not exist`);
1338
+ }
1339
+ mkdirSync(destDir, { recursive: true });
1340
+ const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
1341
+ try {
1342
+ vacuumInto(source, temporary);
1181
1343
  } catch (err) {
1182
1344
  // Never leave a half-written snapshot on disk.
1183
1345
  rmSync(temporary, { force: true });
1184
1346
  throw err;
1185
- } finally {
1186
- connection?.close();
1187
1347
  }
1188
- return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp()}`);
1348
+ return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp(at)}`);
1349
+ }
1350
+
1351
+ /**
1352
+ * How many conductor.db snapshots the daemon's daily cadence keeps — a week
1353
+ * of ledger history at one snapshot per day. Deliberately a fixed count, not
1354
+ * an age or space policy: the store's ledger is append-only and the issue
1355
+ * forbids inventing a fleet-wide retention policy, so pruning is
1356
+ * configured-count only (#289).
1357
+ */
1358
+ export const DB_SNAPSHOT_RETENTION = 7;
1359
+
1360
+ /**
1361
+ * Prune a snapshot directory to the newest `keep` conductor.db snapshots.
1362
+ *
1363
+ * Snapshot stems embed the publish timestamp (`YYYY-MM-DDTHH-MM-SS…`), so
1364
+ * lexicographic order is chronological; only files named with the shared
1365
+ * `DB_SNAPSHOT_STEM` prefix are considered, never unrelated backups next to
1366
+ * them. Idempotent: a directory already at or under the bound removes
1367
+ * nothing. Returns the number of files removed.
1368
+ */
1369
+ export function pruneDbSnapshots(dir: string, keep: number): number {
1370
+ const names = readdirSync(dir)
1371
+ .filter((name) => name.startsWith(DB_SNAPSHOT_STEM))
1372
+ .sort();
1373
+ const stale = names.length > keep ? names.slice(0, names.length - keep) : [];
1374
+ for (const name of stale) unlinkSync(join(dir, name));
1375
+ return stale.length;
1189
1376
  }
1190
1377
 
1191
1378
  /** The UTC calendar day (`YYYY-MM-DD`) a moment falls on — the partition key
@@ -1272,6 +1459,14 @@ export function openStore(dbPath: string): Store {
1272
1459
  db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
1273
1460
  }
1274
1461
  }
1462
+ // Rows written before #737 were never asked whether their tree was
1463
+ // quarantined, and a NULL means exactly that: the pre-quarantine passes
1464
+ // reaped the same trees they always had, so nothing before this release
1465
+ // needs backfilling — the column starts null and only a post-release pass
1466
+ // that actually refuses a broken store writes it.
1467
+ if (!columns.some((column) => column.name === "quarantineDetail")) {
1468
+ db.exec("ALTER TABLE runs ADD COLUMN quarantineDetail TEXT");
1469
+ }
1275
1470
  // Rows written before the settlement audit existed (#128) were never audited,
1276
1471
  // so a NULL here means "no audit ran" and reads the same as "found nothing".
1277
1472
  // That conflation is deliberate and harmless: the flags are advisory, and no
@@ -1306,6 +1501,13 @@ export function openStore(dbPath: string): Store {
1306
1501
  if (!columns.some((column) => column.name === "model")) {
1307
1502
  db.exec("ALTER TABLE runs ADD COLUMN model TEXT");
1308
1503
  }
1504
+ // Resume provenance (#567): rows written before #564 landed were never
1505
+ // resumed, so NULL is the honest reading of "no resume happened" — never
1506
+ // "resumed from unknown". No backfill: the identity existed only in the
1507
+ // dispatch decision, and deriving it now would guess at history.
1508
+ if (!columns.some((column) => column.name === "resumedFromRunId")) {
1509
+ db.exec("ALTER TABLE runs ADD COLUMN resumedFromRunId TEXT");
1510
+ }
1309
1511
  // The count of in-session provider 429s a run recorded was first written by
1310
1512
  // the worker (#573). Rows predating the column are NULL, which is the honest
1311
1513
  // reading: the count was not recorded, so the classifier must not treat the
@@ -1465,10 +1667,10 @@ export function openStore(dbPath: string): Store {
1465
1667
  const insertRun = db.query<unknown, SqlValue[]>(
1466
1668
  `INSERT INTO runs (
1467
1669
  id, project, issue, repo, branch, worktree, state, attempt, turns,
1468
- maxTurns, spendUsd, sessionFile, prUrl, headSha, mergeSha, baseRef,
1670
+ maxTurns, spendUsd, sessionFile, resumedFromRunId, prUrl, headSha, mergeSha, baseRef,
1469
1671
  baseCheck, baseCheckAt, salvageSha, salvageError, salvageAckAt, startedAt,
1470
1672
  endedAt, lastError, settlementFlags, report
1471
- ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1673
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
1472
1674
  );
1473
1675
  const selectRun = db.query<RunRow, [string]>(`SELECT * FROM runs WHERE id = ?`);
1474
1676
  const selectActive = db.query<RunRow, SqlValue[]>(
@@ -1644,6 +1846,56 @@ export function openStore(dbPath: string): Store {
1644
1846
  ORDER BY startedAt DESC, rowid DESC
1645
1847
  LIMIT ?`,
1646
1848
  );
1849
+ // The settled rows a classifier correction can no longer reach (#638). A
1850
+ // `ci-deterministic`/`escalate` row never re-enters `selectUnclassified`
1851
+ // (its class is set and its recovery is not retryable), so evidence the
1852
+ // classifier learned after the row was written — the codeload setup 429 of
1853
+ // #177 — stays a charged attempt forever. This WHERE is the historical
1854
+ // reconciliation's exact predicate: nothing but rows currently reading
1855
+ // `ci-deterministic`, never a class a human re-read, and the class itself is
1856
+ // only ever produced for `state = 'failed'`. Bounded like every other sweep
1857
+ // read, because each candidate costs tracker calls to re-fetch its evidence.
1858
+ // `after` is the persisted review cursor: candidates are only rows strictly
1859
+ // older than it, so a bounded pass progresses through a long misclassified
1860
+ // history instead of rescanning the newest non-matches forever. A NULL cursor
1861
+ // (first pass, or one rendered stale by a classifier-signature change) means
1862
+ // "start from the newest". `SELECT rowid, *` exposes the hidden tie-breaker
1863
+ // so the pass can record an exact resume boundary.
1864
+ const selectHistoricalInfraCandidates = db.query<RunRow & { rowid: number }, SqlValue[]>(
1865
+ `SELECT rowid, * FROM runs
1866
+ WHERE project = ? AND failureClass = 'ci-deterministic' AND state = 'failed'
1867
+ AND (? IS NULL OR (startedAt < ? OR (startedAt = ? AND rowid < ?)))
1868
+ ORDER BY startedAt DESC, rowid DESC
1869
+ LIMIT ?`,
1870
+ );
1871
+ const selectHistoricalInfraCursor = db.query<
1872
+ { cursorStartedAt: number; cursorRowid: number; classifierVersion: string },
1873
+ [string]
1874
+ >(
1875
+ `SELECT cursorStartedAt, cursorRowid, classifierVersion FROM historical_infra_review
1876
+ WHERE project = ?`,
1877
+ );
1878
+ const upsertHistoricalInfraCursor = db.query<
1879
+ unknown,
1880
+ [string, number, number, string, number]
1881
+ >(
1882
+ `INSERT INTO historical_infra_review
1883
+ (project, cursorStartedAt, cursorRowid, classifierVersion, updatedAt)
1884
+ VALUES (?, ?, ?, ?, ?)
1885
+ ON CONFLICT(project) DO UPDATE SET
1886
+ cursorStartedAt = excluded.cursorStartedAt,
1887
+ cursorRowid = excluded.cursorRowid,
1888
+ classifierVersion = excluded.classifierVersion,
1889
+ updatedAt = excluded.updatedAt`,
1890
+ );
1891
+ // The one mutation the historical reconciliation may make. Guarded by the
1892
+ // current class, so a second pass is a no-op and a race can never reclassify
1893
+ // a row a human has re-read. Deliberately only `failureClass` — never
1894
+ // `recoveryAction` or `recoveredAt`: relieving the budget is the point,
1895
+ // re-animating a months-old run into the requeue sweep is not (#638).
1896
+ const reclassifyToInfra = db.query<unknown, [string]>(
1897
+ `UPDATE runs SET failureClass = 'ci-infra' WHERE id = ? AND failureClass = 'ci-deterministic'`,
1898
+ );
1647
1899
  const selectFailureClassCounts = db.query<{ cls: string; n: number }, [string]>(
1648
1900
  `SELECT failureClass AS cls, COUNT(*) AS n FROM runs
1649
1901
  WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
@@ -1671,6 +1923,21 @@ export function openStore(dbPath: string): Store {
1671
1923
  AND (salvageSha IS NOT NULL OR salvageError IS NOT NULL)
1672
1924
  ORDER BY issue ASC`,
1673
1925
  );
1926
+ // Quarantined retained trees (#737), newest attempt per issue — the same
1927
+ // shape as `selectSalvaged` because the same change can land twice: a row
1928
+ // long since reaped must not keep its issue looking quarantined, and an
1929
+ // older quarantined attempt must not claim the issue while a newer,
1930
+ // healthy one replaced it.
1931
+ const selectQuarantined = db.query<RunRow, [string]>(
1932
+ `SELECT * FROM runs
1933
+ WHERE rowid IN (
1934
+ SELECT MAX(rowid) FROM runs
1935
+ WHERE project = ?
1936
+ GROUP BY issue
1937
+ )
1938
+ AND quarantineDetail IS NOT NULL
1939
+ ORDER BY issue ASC`,
1940
+ );
1674
1941
  // Newest attempt for one issue. `startedAt` is millisecond-resolution and two
1675
1942
  // attempts could in principle share one, so rowid breaks the tie by insertion
1676
1943
  // order — a `tail` that attached to the older of two same-millisecond attempts
@@ -2319,6 +2586,72 @@ export function openStore(dbPath: string): Store {
2319
2586
  },
2320
2587
  );
2321
2588
 
2589
+ // The durable review-revision request outbox (#677): the orchestrator's verb
2590
+ // writes the row (findings, head, round, target session) and the daemon's
2591
+ // next dispatch pass wakes it. The duplicate in-flight guard is the insert:
2592
+ // a pending row for the same run refuses the second request atomically, and
2593
+ // once dispatched the run row's own state (pushed-green → running) is the
2594
+ // in-flight marker.
2595
+ const insertReviewRevision = db.query<unknown, SqlValue[]>(
2596
+ `INSERT INTO review_revisions
2597
+ (id, project, runId, issue, prUrl, headSha, findings, round, reason, sessionFile, requestedAt, dispatchedAt, settledAt, outcome)
2598
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL)`,
2599
+ );
2600
+ const selectPendingReviewRevisions = db.query<ReviewRevisionRow, [string]>(
2601
+ `SELECT * FROM review_revisions
2602
+ WHERE project = ? AND dispatchedAt IS NULL AND settledAt IS NULL
2603
+ ORDER BY requestedAt ASC, rowid ASC`,
2604
+ );
2605
+ const selectPendingReviewForRun = db.query<ReviewRevisionRow, [string, string]>(
2606
+ `SELECT * FROM review_revisions
2607
+ WHERE project = ? AND runId = ? AND dispatchedAt IS NULL AND settledAt IS NULL
2608
+ LIMIT 1`,
2609
+ );
2610
+ const selectLatestReviewRound = db.query<{ n: number | null }, [string, string]>(
2611
+ `SELECT MAX(round) AS n FROM review_revisions WHERE project = ? AND runId = ?`,
2612
+ );
2613
+ const markReviewRevisionDispatchedRow = db.query<unknown, [number, string]>(
2614
+ `UPDATE review_revisions SET dispatchedAt = ? WHERE id = ?`,
2615
+ );
2616
+ const settleReviewRevisionRow = db.query<unknown, [string, number, string]>(
2617
+ `UPDATE review_revisions SET outcome = ?, settledAt = ? WHERE id = ?`,
2618
+ );
2619
+ const selectUnsettledReviewRevisions = db.query<ReviewRevisionRow, [string]>(
2620
+ `SELECT * FROM review_revisions
2621
+ WHERE project = ? AND settledAt IS NULL
2622
+ ORDER BY requestedAt ASC, rowid ASC`,
2623
+ );
2624
+ const requeueReviewRevisionRow = db.query<unknown, [string]>(
2625
+ `UPDATE review_revisions SET dispatchedAt = NULL WHERE id = ?`,
2626
+ );
2627
+ const claimRunForReviewRow = db.query<unknown, [string]>(
2628
+ `UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'pushed-green'`,
2629
+ );
2630
+
2631
+ // The claim and the insert share a transaction so two concurrent revision
2632
+ // requests for one run cannot both win: the second sees the first's pending
2633
+ // row and returns undefined, which the verb turns into the in-flight refusal.
2634
+ const claimReviewRevision = db.transaction(
2635
+ (draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined => {
2636
+ if (selectPendingReviewForRun.get(draft.project, draft.runId) !== null) return undefined;
2637
+ const record: ReviewRevisionRecord = { ...draft, id: crypto.randomUUID() };
2638
+ insertReviewRevision.run(
2639
+ record.id,
2640
+ record.project,
2641
+ record.runId,
2642
+ record.issue,
2643
+ record.prUrl,
2644
+ record.headSha,
2645
+ record.findings,
2646
+ record.round,
2647
+ record.reason,
2648
+ record.sessionFile ?? null,
2649
+ record.requestedAt,
2650
+ );
2651
+ return record;
2652
+ },
2653
+ );
2654
+
2322
2655
  // The label projection outbox (#201). A decided label change is a row until
2323
2656
  // the tracker has applied it; the projector drains in id order, and one
2324
2657
  // issue's ops are atomic as a run — a failed op parks the rest of its issue
@@ -2443,6 +2776,7 @@ export function openStore(dbPath: string): Store {
2443
2776
  record.maxTurns,
2444
2777
  record.spendUsd,
2445
2778
  toSql(record.sessionFile),
2779
+ toSql(record.resumedFromRunId),
2446
2780
  toSql(record.prUrl),
2447
2781
  toSql(record.headSha),
2448
2782
  toSql(record.mergeSha),
@@ -2526,6 +2860,30 @@ export function openStore(dbPath: string): Store {
2526
2860
  runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[] {
2527
2861
  return selectRunsForPr.all(project, prUrl, mergedSinceEpochMs).map(toRecord);
2528
2862
  },
2863
+ createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined {
2864
+ return claimReviewRevision(draft);
2865
+ },
2866
+ pendingReviewRevisions(project: string): ReviewRevisionRecord[] {
2867
+ return selectPendingReviewRevisions.all(project).map(toReviewRevision);
2868
+ },
2869
+ latestReviewRound(project: string, runId: string): number {
2870
+ return selectLatestReviewRound.get(project, runId)?.n ?? 0;
2871
+ },
2872
+ markReviewRevisionDispatched(id: string, at: number): void {
2873
+ markReviewRevisionDispatchedRow.run(at, id);
2874
+ },
2875
+ settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void {
2876
+ settleReviewRevisionRow.run(outcome, at, id);
2877
+ },
2878
+ unsettledReviewRevisions(project: string): ReviewRevisionRecord[] {
2879
+ return selectUnsettledReviewRevisions.all(project).map(toReviewRevision);
2880
+ },
2881
+ requeueReviewRevision(id: string): void {
2882
+ requeueReviewRevisionRow.run(id);
2883
+ },
2884
+ claimRunForReview(runId: string): boolean {
2885
+ return claimRunForReviewRow.run(runId).changes > 0;
2886
+ },
2529
2887
  runsNeedingBaseCheck(project: string, limit = 20): RunRecord[] {
2530
2888
  return selectPendingBaseChecks.all(project, limit).map(toRecord);
2531
2889
  },
@@ -2596,6 +2954,10 @@ export function openStore(dbPath: string): Store {
2596
2954
  return selectSalvaged.all(project).map(toRecord);
2597
2955
  },
2598
2956
 
2957
+ quarantinedRuns(project: string): RunRecord[] {
2958
+ return selectQuarantined.all(project).map(toRecord);
2959
+ },
2960
+
2599
2961
  attemptsFor(project: string, issue: number): number {
2600
2962
  return countAttempts.get(project, issue)?.n ?? 0;
2601
2963
  },
@@ -3066,6 +3428,38 @@ export function openStore(dbPath: string): Store {
3066
3428
  return selectUnclassified.all(project, limit).map(toRecord);
3067
3429
  },
3068
3430
 
3431
+ historicalInfraCandidates(
3432
+ project: string,
3433
+ limit = 20,
3434
+ after?: { startedAt: number; rowid: number },
3435
+ ): HistoricalInfraCandidate[] {
3436
+ const cursorAt = after?.startedAt ?? null;
3437
+ const cursorRowid = after?.rowid ?? null;
3438
+ return selectHistoricalInfraCandidates
3439
+ .all(project, cursorAt, cursorAt, cursorAt, cursorRowid, limit)
3440
+ .map((row) => ({ ...toRecord(row), rowid: row.rowid }));
3441
+ },
3442
+
3443
+ historicalInfraCursor(project: string) {
3444
+ const row = selectHistoricalInfraCursor.get(project);
3445
+ return row == null
3446
+ ? undefined
3447
+ : { startedAt: row.cursorStartedAt, rowid: row.cursorRowid, classifierVersion: row.classifierVersion };
3448
+ },
3449
+
3450
+ setHistoricalInfraCursor(
3451
+ project: string,
3452
+ startedAt: number,
3453
+ rowid: number,
3454
+ classifierVersion: string,
3455
+ ): void {
3456
+ upsertHistoricalInfraCursor.run(project, startedAt, rowid, classifierVersion, Date.now());
3457
+ },
3458
+
3459
+ reclassifyInfra(id: string): boolean {
3460
+ return reclassifyToInfra.run(id).changes > 0;
3461
+ },
3462
+
3069
3463
  failureClassCounts(project: string): { cls: FailureClass; n: number }[] {
3070
3464
  return selectFailureClassCounts
3071
3465
  .all(project)