omp-conductor 0.16.2 → 0.17.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +38 -4
  2. package/REFERENCE.md +18 -12
  3. package/package.json +2 -1
  4. package/schema/config.schema.json +16 -0
  5. package/src/admission.ts +159 -43
  6. package/src/availability.ts +27 -1
  7. package/src/briefs/worker.md +2 -0
  8. package/src/clack-ui.ts +83 -0
  9. package/src/command-manifest.ts +16 -7
  10. package/src/commands/arm.ts +11 -3
  11. package/src/commands/decision.ts +17 -7
  12. package/src/commands/doctor.ts +18 -1
  13. package/src/commands/hold.ts +9 -7
  14. package/src/commands/ledger.ts +25 -4
  15. package/src/commands/message.ts +32 -4
  16. package/src/commands/setup.ts +61 -10
  17. package/src/commands/stats.ts +9 -5
  18. package/src/commands/status.ts +32 -5
  19. package/src/commands/tail.ts +13 -1
  20. package/src/commands/watch.ts +16 -7
  21. package/src/config-schema.ts +20 -0
  22. package/src/config.ts +37 -0
  23. package/src/daemon.ts +1240 -18
  24. package/src/doctor.ts +310 -22
  25. package/src/escalate.ts +560 -57
  26. package/src/failure-class.ts +56 -13
  27. package/src/fleet.ts +224 -47
  28. package/src/gitops.ts +103 -24
  29. package/src/lifecycle.ts +7 -2
  30. package/src/orchestrator-tick.ts +372 -157
  31. package/src/privileged.ts +3 -0
  32. package/src/release-policy.ts +177 -5
  33. package/src/setup-answers.ts +135 -0
  34. package/src/setup-host.ts +193 -4
  35. package/src/setup-install.ts +2 -0
  36. package/src/setup-probe.ts +1 -0
  37. package/src/setup-wizard.ts +1296 -101
  38. package/src/setup.ts +60 -3
  39. package/src/status-render.ts +11 -1
  40. package/src/store.ts +333 -12
  41. package/src/tracker/github.ts +562 -13
  42. package/src/types.ts +204 -2
  43. package/src/ui/progress.ts +32 -0
  44. package/src/ui/style.ts +11 -0
  45. package/src/upgrade.ts +50 -19
  46. package/src/verbs/actions.ts +66 -18
  47. package/src/verbs/protocol.ts +45 -0
  48. package/src/verbs/server.ts +212 -11
  49. package/src/wizard-ui.ts +14 -5
  50. package/src/worker.ts +26 -0
  51. package/systemd/omp-conductor-recover.sh +73 -0
  52. package/systemd/recover-unit-test.sh +61 -0
package/src/setup.ts CHANGED
@@ -8,7 +8,8 @@
8
8
  * least tested code in the package.
9
9
  *
10
10
  * Two functions here mutate something outside the process: `createMissingLabels`
11
- * and `writeOrchestratorBrief`. Everything else reads, or computes. That is what
11
+ * (and its rollback compensation half, `deleteCreatedLabels`) and
12
+ * `writeOrchestratorBrief`. Everything else reads, or computes. That is what
12
13
  * lets the plugin show a complete plan before asking for consent, and it is a
13
14
  * property worth preserving — check it before adding a function.
14
15
  *
@@ -40,6 +41,7 @@ import {
40
41
  configPath,
41
42
  defaultMirrorRoot,
42
43
  defaultWorkspaceRoot,
44
+ resolveArmProof,
43
45
  resolveCaps,
44
46
  resolvePolicy,
45
47
  resolveReleaseGrants,
@@ -50,6 +52,7 @@ import { graphProjectPath, graphRepos } from "./graph.ts";
50
52
  import { ompSettingsOverlay } from "./omp-settings.ts";
51
53
  import {
52
54
  CONFIG_VERSION,
55
+ DEFAULT_ARM_PROOF,
53
56
  DEFAULT_AUTHORITY,
54
57
  DEFAULT_CAPS,
55
58
  DEFAULT_PROJECT_POLICY,
@@ -57,6 +60,7 @@ import {
57
60
  DENIED_RELEASE_GRANTS,
58
61
  RELEASE_SHAPES,
59
62
  WEEKDAYS,
63
+ type ArmProof,
60
64
  type BaseFreshness,
61
65
  type BehindBaseAction,
62
66
  type AuthorityHolder,
@@ -205,6 +209,12 @@ export interface SetupAnswers {
205
209
  * leave one to be defaulted by whichever reader gets there first.
206
210
  */
207
211
  policy: ProjectPolicy;
212
+ /**
213
+ * How `arm` proves a human just approved arming (#613). Always complete:
214
+ * the wizard asks about it in the policy area, so an answers object can never
215
+ * leave it to be defaulted by whichever reader gets there first.
216
+ */
217
+ armProof: ArmProof;
208
218
  /**
209
219
  * Hand-edited recovery merge authorizations carried through setup unchanged.
210
220
  * The wizard never grants one; forgetting them during an unrelated amend
@@ -381,8 +391,23 @@ export const BEHIND_BASE_CHOICES: { readonly [K in BehindBaseAction]: string } =
381
391
  escalate: "page a human rather than guess",
382
392
  };
383
393
 
394
+ /**
395
+ * What each arming proof means to the operator being asked about it, in their
396
+ * words rather than the gate's (conductor #613). The `claim-only` consequence
397
+ * is the one line the issue contract requires the question to state: anything
398
+ * that can invoke the already-privileged `omp-conductor arm` command can start
399
+ * dispatch once the live claim and poller pass. Mapped over the closed union
400
+ * so a third proof fails to compile here instead of reaching a wizard with no
401
+ * question for it.
402
+ */
403
+ export const ARM_PROOF_CHOICES: { readonly [K in ArmProof]: string } = {
404
+ challenge: "an authenticated Telegram challenge proves a human just approved arming",
405
+ "claim-only": "anyone who can invoke the already-privileged `omp-conductor arm` command can start dispatch once the live claim and poller pass",
406
+ };
407
+
384
408
  export const RELEASE_REQUIREMENT_CHOICES: { readonly [K in ReleaseRequirement]: string } = {
385
- "runs-settled": "every run this release covers actually merged, not merely reached a green PR",
409
+ "runs-settled": "every run of the released repo actually merged, not merely reached a green PR",
410
+ "fleet-runs-settled": "every run in the project actually merged — suite-wide strictness for shapes that consume several repos",
386
411
  "no-open-prs": "no pull request is still open against the branch being released",
387
412
  "queue-drained": "nothing still carries the queue label",
388
413
  "base-branch-green": "the newest observed post-merge base-branch workflows are green",
@@ -700,11 +725,33 @@ export async function createMissingLabels(trackerRepo: string, plan: LabelPlan[]
700
725
  continue;
701
726
  }
702
727
  if (/already exists/i.test(r.stderr)) continue;
703
- throw new Error(`Could not create label "${label.name}" in ${trackerRepo}: ${briefly(r.stderr)}`);
728
+ // The labels created before this one failed ride on the error: the setup
729
+ // transaction compensates the tracker exactly, and a throw without the
730
+ // partial list would orphan them (#652).
731
+ throw Object.assign(
732
+ new Error(`Could not create label "${label.name}" in ${trackerRepo}: ${briefly(r.stderr)}`),
733
+ { created },
734
+ );
704
735
  }
705
736
  return created;
706
737
  }
707
738
 
739
+ /**
740
+ * Compensation half of {@link createMissingLabels}: deletes exactly the labels
741
+ * a failed setup apply created, so the tracker returns to its pre-entry state.
742
+ * Runs only on rollback, where a deletion failure is a reported restoration
743
+ * failure — the orphaned label is named, never silently left behind (#652).
744
+ */
745
+ export async function deleteCreatedLabels(trackerRepo: string, created: string[]): Promise<void> {
746
+ const failures: string[] = [];
747
+ for (const name of created) {
748
+ const r = await gh(["label", "delete", name, "--repo", trackerRepo, "--yes"]);
749
+ if (r.code === 0 || /not found/i.test(r.stderr)) continue;
750
+ failures.push(`could not delete label "${name}" in ${trackerRepo}: ${briefly(r.stderr)}`);
751
+ }
752
+ if (failures.length > 0) throw new Error(failures.join("; "));
753
+ }
754
+
708
755
  const QUIET_INTERRUPT_ON: InterruptCategory[] = [
709
756
  "tier2",
710
757
  "fleet-stopped",
@@ -877,6 +924,10 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
877
924
  // Written out in full for the same reason: the file then says what a merge
878
925
  // and a release require without anyone having to know a default (#129).
879
926
  policy: clonePolicy(a.policy),
927
+ // Written out even when it is the default, so an operator amending the
928
+ // policy has a line in the file to point at — and the recovery playbook
929
+ // can read which proof the project opted into (#613).
930
+ arm: { proof: a.armProof },
880
931
  ...(a.recoveryMerges === undefined
881
932
  ? {}
882
933
  : { recoveryMerges: a.recoveryMerges.map((entry) => ({ ...entry })) }),
@@ -964,6 +1015,9 @@ export function defaultAnswers(projectName: string, opts: { added?: boolean } =
964
1015
  authority: { ...SETUP_DEFAULTS.authority },
965
1016
  releaseGrants: { ...SETUP_DEFAULTS.releaseGrants },
966
1017
  policy: clonePolicy(SETUP_DEFAULTS.policy),
1018
+ // The strict reading, matching the loader's absent-key default: a project
1019
+ // that never answered keeps today's authenticated challenge round-trip.
1020
+ armProof: DEFAULT_ARM_PROOF,
967
1021
  orchestratorMode: SETUP_DEFAULTS.orchestratorMode,
968
1022
  reportScope: SETUP_DEFAULT_REPORT_SCOPE,
969
1023
  writeOrchestratorBrief: false,
@@ -1060,6 +1114,9 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
1060
1114
  authority: { ...p.authority },
1061
1115
  releaseGrants: resolveReleaseGrants(p),
1062
1116
  policy: resolvePolicy(p),
1117
+ // The loader materialises `arm` complete, so this is the answer the project
1118
+ // actually has — never a default guessed at the amend prompt.
1119
+ armProof: resolveArmProof(p),
1063
1120
  orchestratorMode: p.escalation.orchestrator,
1064
1121
  reportScope: reportScopeFromPolicy(p.reporting),
1065
1122
  writeOrchestratorBrief: false,
@@ -415,7 +415,17 @@ function formatProjectBody(
415
415
  lines.push("active runs");
416
416
  for (const r of s.activeRuns) {
417
417
  const phase = workerPhases.get(r.issue);
418
- const state = phase === "pausing" || phase === "paused" ? phase : r.state;
418
+ // A run in a live review round reads `review-revision N`, distinct from
419
+ // a failure and from an ordinary continuation, with the round number
420
+ // from the durable revision row (#692). The live pause overlay still
421
+ // wins: an operator pause is the newer fact about the same session.
422
+ const round = s.reviewRounds?.[r.id];
423
+ const state =
424
+ phase === "pausing" || phase === "paused"
425
+ ? phase
426
+ : round !== undefined
427
+ ? `review-revision ${round}`
428
+ : r.state;
419
429
  lines.push(
420
430
  ` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
421
431
  `${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
package/src/store.ts CHANGED
@@ -41,6 +41,7 @@ import type {
41
41
  FrictionKind,
42
42
  FrictionObservation,
43
43
  FrictionSignal,
44
+ HistoricalInfraCandidate,
44
45
  HeldNotice,
45
46
  InterruptCategory,
46
47
  HeldNoticeDraft,
@@ -60,6 +61,9 @@ import type {
60
61
  ReportKind,
61
62
  ReportRecord,
62
63
  RecoveryAction,
64
+ ReviewReason,
65
+ ReviewRevisionOutcome,
66
+ ReviewRevisionRecord,
63
67
  RunPatch,
64
68
  RunRecord,
65
69
  RunState,
@@ -99,6 +103,7 @@ const FRICTION_HOLD_REASONS: ReadonlySet<FrictionAdmissionReason> = new Set([
99
103
  "failed-attempts",
100
104
  "continuations",
101
105
  "stale-base",
106
+ "critical-base-verify-error",
102
107
  "parent-lookup-error",
103
108
  "issue-state-lookup-error",
104
109
  "open-pr-lookup-error",
@@ -361,6 +366,48 @@ interface MergeLockRow {
361
366
  at: number;
362
367
  }
363
368
 
369
+ /** The `review_revisions` table exactly as SQLite hands it back (#677). */
370
+ interface ReviewRevisionRow {
371
+ id: string;
372
+ project: string;
373
+ runId: string;
374
+ issue: number;
375
+ prUrl: string;
376
+ headSha: string;
377
+ findings: string;
378
+ round: number;
379
+ reason: string;
380
+ sessionFile: string | null;
381
+ requestedAt: number;
382
+ dispatchedAt: number | null;
383
+ settledAt: number | null;
384
+ outcome: string | null;
385
+ }
386
+
387
+ /**
388
+ * NULL columns become absent properties, matching the other row converters:
389
+ * a record read back out of the store deep-equals the one that went in.
390
+ */
391
+ function toReviewRevision(row: ReviewRevisionRow): ReviewRevisionRecord {
392
+ const record: ReviewRevisionRecord = {
393
+ id: row.id,
394
+ project: row.project,
395
+ runId: row.runId,
396
+ issue: row.issue,
397
+ prUrl: row.prUrl,
398
+ headSha: row.headSha,
399
+ findings: row.findings,
400
+ round: row.round,
401
+ reason: row.reason as ReviewReason,
402
+ requestedAt: row.requestedAt,
403
+ };
404
+ if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
405
+ if (row.dispatchedAt !== null) record.dispatchedAt = row.dispatchedAt;
406
+ if (row.settledAt !== null) record.settledAt = row.settledAt;
407
+ if (row.outcome !== null) record.outcome = row.outcome as ReviewRevisionOutcome;
408
+ return record;
409
+ }
410
+
364
411
  /** The `intake_items` table exactly as SQLite hands it back (#299). */
365
412
  interface IntakeRow {
366
413
  id: string;
@@ -492,6 +539,22 @@ CREATE TABLE IF NOT EXISTS runs (
492
539
  CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
493
540
  CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
494
541
 
542
+ -- The historical infrastructure reconciliation's per-project review cursor
543
+ -- (#638). A bounded pass examines settled ci-deterministic rows newest-first
544
+ -- and persists the boundary of the last row it definitively decided, so older
545
+ -- repairable rows are reached across later daemon starts instead of being
546
+ -- starved by repeated rescans of the newest non-matches. classifierVersion is
547
+ -- the signature-list fingerprint at the time the cursor advanced; when the
548
+ -- classifier learns a new signature the stamp goes stale and the pass restarts
549
+ -- from the newest row rather than skipping past newly recognised evidence.
550
+ CREATE TABLE IF NOT EXISTS historical_infra_review (
551
+ project TEXT PRIMARY KEY,
552
+ cursorStartedAt INTEGER NOT NULL,
553
+ cursorRowid INTEGER NOT NULL,
554
+ classifierVersion TEXT NOT NULL,
555
+ updatedAt INTEGER NOT NULL
556
+ );
557
+
495
558
  CREATE TABLE IF NOT EXISTS base_health (
496
559
  project TEXT NOT NULL,
497
560
  repo TEXT NOT NULL,
@@ -693,6 +756,31 @@ CREATE TABLE IF NOT EXISTS merge_locks (
693
756
  at INTEGER NOT NULL
694
757
  );
695
758
 
759
+ -- One durable review-revision request (#677): an orchestrator returned a
760
+ -- green, run-owned pull request to its worker with blocking findings. Every
761
+ -- field the daemon needs to resume the exact session is persisted BEFORE the
762
+ -- worker is woken -- the findings, the exact reviewed head, the round number,
763
+ -- and the target run/session -- and the row doubles as the duplicate guard: a
764
+ -- revision whose row exists pending is one nobody may queue again.
765
+ CREATE TABLE IF NOT EXISTS review_revisions (
766
+ id TEXT PRIMARY KEY,
767
+ project TEXT NOT NULL,
768
+ runId TEXT NOT NULL,
769
+ issue INTEGER NOT NULL,
770
+ prUrl TEXT NOT NULL,
771
+ headSha TEXT NOT NULL,
772
+ findings TEXT NOT NULL,
773
+ round INTEGER NOT NULL,
774
+ reason TEXT NOT NULL,
775
+ sessionFile TEXT,
776
+ requestedAt INTEGER NOT NULL,
777
+ dispatchedAt INTEGER,
778
+ settledAt INTEGER,
779
+ outcome TEXT
780
+ );
781
+ CREATE INDEX IF NOT EXISTS review_revisions_pending
782
+ ON review_revisions (project, dispatchedAt, runId);
783
+
696
784
  -- Questions the orchestrator has put to its operator, and their answers (#136).
697
785
  --
698
786
  -- The reason this is a table and not the session's memory: a question lived only
@@ -1143,12 +1231,59 @@ export function dbPath(): string {
1143
1231
  return join(stateDir(), "conductor.db");
1144
1232
  }
1145
1233
 
1234
+ /**
1235
+ * Live worker count across the run store, opening the database WITHOUT
1236
+ * creating or upgrading it. The setup apply's quiescence preflight must count
1237
+ * workers before the acknowledged barrier, and database preparation is itself
1238
+ * one of the mutations that barrier must precede — so the observation that
1239
+ * gates it cannot be the thing that creates or migrates the store (#618). The
1240
+ * on-disk database is opened read-only (never created, never "prepared"); a
1241
+ * missing file reports zero because no run has ever been persisted, which the
1242
+ * caller only trusts when no daemon is running either. An unreadable or
1243
+ * schema-broken file throws rather than reading as zero — fail closed, never
1244
+ * absence.
1245
+ */
1246
+ export function liveWorkersReadOnly(project?: string): number {
1247
+ const path = dbPath();
1248
+ if (!existsSync(path)) return 0;
1249
+ let db: Database | undefined;
1250
+ try {
1251
+ db = new Database(path, { create: false, readonly: true });
1252
+ db.exec("PRAGMA busy_timeout = 5000;");
1253
+ const placeholders = LIVE_STATES.map(() => "?").join(", ");
1254
+ // `undefined` is the host-global selector: every project's live rows count,
1255
+ // never only the rows of a project literally named "" — a host-global
1256
+ // observation that scoped to `project = ''` hid every real named row and
1257
+ // read zero beside a running fleet (#651 review #3).
1258
+ const row =
1259
+ project === undefined
1260
+ ? db
1261
+ .query<{ n: number }, SqlValue[]>(
1262
+ `SELECT COUNT(*) AS n FROM runs WHERE state IN (${placeholders})`,
1263
+ )
1264
+ .get(...LIVE_STATES)
1265
+ : db
1266
+ .query<{ n: number }, SqlValue[]>(
1267
+ `SELECT COUNT(*) AS n FROM runs WHERE project = ? AND state IN (${placeholders})`,
1268
+ )
1269
+ .get(project, ...LIVE_STATES);
1270
+ return row?.n ?? 0;
1271
+ } catch (err) {
1272
+ throw new Error(
1273
+ `cannot read ${path} to prove the fleet is quiescent: ${err instanceof Error ? err.message : String(err)}`,
1274
+ );
1275
+ } finally {
1276
+ db?.close();
1277
+ }
1278
+ }
1279
+
1146
1280
  /** The stem every restorable `conductor.db` snapshot is published under. */
1147
1281
  export const DB_SNAPSHOT_STEM = "conductor.db.bak";
1148
1282
 
1149
1283
  /**
1150
- * A fresh, restorable copy of the store at `source`, published under `destDir`
1151
- * with the same temp-then-atomic-publish convention as the config backups.
1284
+ * A single-file, transaction-consistent copy of the store at `source` into
1285
+ * `target` the shared core of {@link snapshotDb} and the setup apply's
1286
+ * rollback capture.
1152
1287
  *
1153
1288
  * Uses SQLite's own `VACUUM INTO`, never a plain file copy: the store opens
1154
1289
  * with `journal_mode=WAL`, and a `copyFileSync` can capture a torn database
@@ -1158,10 +1293,30 @@ export const DB_SNAPSHOT_STEM = "conductor.db.bak";
1158
1293
  * writes a single-file snapshot, so the result opens cleanly in a fresh
1159
1294
  * connection and contains every committed row.
1160
1295
  *
1161
- * The snapshot runs against its own read of the source, deliberately not the
1162
- * caller's writer connection, so a snapshot can be taken while a writer holds
1163
- * an uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
1296
+ * Runs against its own read of the source, deliberately not the caller's
1297
+ * writer connection, so a snapshot can be taken while a writer holds an
1298
+ * uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
1164
1299
  * refused.
1300
+ */
1301
+ export function vacuumInto(source: string, target: string): void {
1302
+ if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
1303
+ if (!existsSync(source)) {
1304
+ throw new Error(`cannot snapshot ${source}: the store does not exist`);
1305
+ }
1306
+ let connection: Database | undefined;
1307
+ try {
1308
+ connection = new Database(source);
1309
+ connection.exec("PRAGMA busy_timeout = 5000;");
1310
+ // `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
1311
+ connection.exec(`VACUUM INTO '${target.replaceAll("'", "''")}'`);
1312
+ } finally {
1313
+ connection?.close();
1314
+ }
1315
+ }
1316
+
1317
+ /**
1318
+ * A fresh, restorable copy of the store at `source`, published under `destDir`
1319
+ * with the same temp-then-atomic-publish convention as the config backups.
1165
1320
  *
1166
1321
  * Returns the published snapshot path.
1167
1322
  */
@@ -1172,18 +1327,12 @@ export function snapshotDb(source: string, destDir: string): string {
1172
1327
  }
1173
1328
  mkdirSync(destDir, { recursive: true });
1174
1329
  const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
1175
- let connection: Database | undefined;
1176
1330
  try {
1177
- connection = new Database(source);
1178
- connection.exec("PRAGMA busy_timeout = 5000;");
1179
- // `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
1180
- connection.exec(`VACUUM INTO '${temporary.replaceAll("'", "''")}'`);
1331
+ vacuumInto(source, temporary);
1181
1332
  } catch (err) {
1182
1333
  // Never leave a half-written snapshot on disk.
1183
1334
  rmSync(temporary, { force: true });
1184
1335
  throw err;
1185
- } finally {
1186
- connection?.close();
1187
1336
  }
1188
1337
  return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp()}`);
1189
1338
  }
@@ -1644,6 +1793,56 @@ export function openStore(dbPath: string): Store {
1644
1793
  ORDER BY startedAt DESC, rowid DESC
1645
1794
  LIMIT ?`,
1646
1795
  );
1796
+ // The settled rows a classifier correction can no longer reach (#638). A
1797
+ // `ci-deterministic`/`escalate` row never re-enters `selectUnclassified`
1798
+ // (its class is set and its recovery is not retryable), so evidence the
1799
+ // classifier learned after the row was written — the codeload setup 429 of
1800
+ // #177 — stays a charged attempt forever. This WHERE is the historical
1801
+ // reconciliation's exact predicate: nothing but rows currently reading
1802
+ // `ci-deterministic`, never a class a human re-read, and the class itself is
1803
+ // only ever produced for `state = 'failed'`. Bounded like every other sweep
1804
+ // read, because each candidate costs tracker calls to re-fetch its evidence.
1805
+ // `after` is the persisted review cursor: candidates are only rows strictly
1806
+ // older than it, so a bounded pass progresses through a long misclassified
1807
+ // history instead of rescanning the newest non-matches forever. A NULL cursor
1808
+ // (first pass, or one rendered stale by a classifier-signature change) means
1809
+ // "start from the newest". `SELECT rowid, *` exposes the hidden tie-breaker
1810
+ // so the pass can record an exact resume boundary.
1811
+ const selectHistoricalInfraCandidates = db.query<RunRow & { rowid: number }, SqlValue[]>(
1812
+ `SELECT rowid, * FROM runs
1813
+ WHERE project = ? AND failureClass = 'ci-deterministic' AND state = 'failed'
1814
+ AND (? IS NULL OR (startedAt < ? OR (startedAt = ? AND rowid < ?)))
1815
+ ORDER BY startedAt DESC, rowid DESC
1816
+ LIMIT ?`,
1817
+ );
1818
+ const selectHistoricalInfraCursor = db.query<
1819
+ { cursorStartedAt: number; cursorRowid: number; classifierVersion: string },
1820
+ [string]
1821
+ >(
1822
+ `SELECT cursorStartedAt, cursorRowid, classifierVersion FROM historical_infra_review
1823
+ WHERE project = ?`,
1824
+ );
1825
+ const upsertHistoricalInfraCursor = db.query<
1826
+ unknown,
1827
+ [string, number, number, string, number]
1828
+ >(
1829
+ `INSERT INTO historical_infra_review
1830
+ (project, cursorStartedAt, cursorRowid, classifierVersion, updatedAt)
1831
+ VALUES (?, ?, ?, ?, ?)
1832
+ ON CONFLICT(project) DO UPDATE SET
1833
+ cursorStartedAt = excluded.cursorStartedAt,
1834
+ cursorRowid = excluded.cursorRowid,
1835
+ classifierVersion = excluded.classifierVersion,
1836
+ updatedAt = excluded.updatedAt`,
1837
+ );
1838
+ // The one mutation the historical reconciliation may make. Guarded by the
1839
+ // current class, so a second pass is a no-op and a race can never reclassify
1840
+ // a row a human has re-read. Deliberately only `failureClass` — never
1841
+ // `recoveryAction` or `recoveredAt`: relieving the budget is the point,
1842
+ // re-animating a months-old run into the requeue sweep is not (#638).
1843
+ const reclassifyToInfra = db.query<unknown, [string]>(
1844
+ `UPDATE runs SET failureClass = 'ci-infra' WHERE id = ? AND failureClass = 'ci-deterministic'`,
1845
+ );
1647
1846
  const selectFailureClassCounts = db.query<{ cls: string; n: number }, [string]>(
1648
1847
  `SELECT failureClass AS cls, COUNT(*) AS n FROM runs
1649
1848
  WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
@@ -2319,6 +2518,72 @@ export function openStore(dbPath: string): Store {
2319
2518
  },
2320
2519
  );
2321
2520
 
2521
+ // The durable review-revision request outbox (#677): the orchestrator's verb
2522
+ // writes the row (findings, head, round, target session) and the daemon's
2523
+ // next dispatch pass wakes it. The duplicate in-flight guard is the insert:
2524
+ // a pending row for the same run refuses the second request atomically, and
2525
+ // once dispatched the run row's own state (pushed-green → running) is the
2526
+ // in-flight marker.
2527
+ const insertReviewRevision = db.query<unknown, SqlValue[]>(
2528
+ `INSERT INTO review_revisions
2529
+ (id, project, runId, issue, prUrl, headSha, findings, round, reason, sessionFile, requestedAt, dispatchedAt, settledAt, outcome)
2530
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL)`,
2531
+ );
2532
+ const selectPendingReviewRevisions = db.query<ReviewRevisionRow, [string]>(
2533
+ `SELECT * FROM review_revisions
2534
+ WHERE project = ? AND dispatchedAt IS NULL AND settledAt IS NULL
2535
+ ORDER BY requestedAt ASC, rowid ASC`,
2536
+ );
2537
+ const selectPendingReviewForRun = db.query<ReviewRevisionRow, [string, string]>(
2538
+ `SELECT * FROM review_revisions
2539
+ WHERE project = ? AND runId = ? AND dispatchedAt IS NULL AND settledAt IS NULL
2540
+ LIMIT 1`,
2541
+ );
2542
+ const selectLatestReviewRound = db.query<{ n: number | null }, [string, string]>(
2543
+ `SELECT MAX(round) AS n FROM review_revisions WHERE project = ? AND runId = ?`,
2544
+ );
2545
+ const markReviewRevisionDispatchedRow = db.query<unknown, [number, string]>(
2546
+ `UPDATE review_revisions SET dispatchedAt = ? WHERE id = ?`,
2547
+ );
2548
+ const settleReviewRevisionRow = db.query<unknown, [string, number, string]>(
2549
+ `UPDATE review_revisions SET outcome = ?, settledAt = ? WHERE id = ?`,
2550
+ );
2551
+ const selectUnsettledReviewRevisions = db.query<ReviewRevisionRow, [string]>(
2552
+ `SELECT * FROM review_revisions
2553
+ WHERE project = ? AND settledAt IS NULL
2554
+ ORDER BY requestedAt ASC, rowid ASC`,
2555
+ );
2556
+ const requeueReviewRevisionRow = db.query<unknown, [string]>(
2557
+ `UPDATE review_revisions SET dispatchedAt = NULL WHERE id = ?`,
2558
+ );
2559
+ const claimRunForReviewRow = db.query<unknown, [string]>(
2560
+ `UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'pushed-green'`,
2561
+ );
2562
+
2563
+ // The claim and the insert share a transaction so two concurrent revision
2564
+ // requests for one run cannot both win: the second sees the first's pending
2565
+ // row and returns undefined, which the verb turns into the in-flight refusal.
2566
+ const claimReviewRevision = db.transaction(
2567
+ (draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined => {
2568
+ if (selectPendingReviewForRun.get(draft.project, draft.runId) !== null) return undefined;
2569
+ const record: ReviewRevisionRecord = { ...draft, id: crypto.randomUUID() };
2570
+ insertReviewRevision.run(
2571
+ record.id,
2572
+ record.project,
2573
+ record.runId,
2574
+ record.issue,
2575
+ record.prUrl,
2576
+ record.headSha,
2577
+ record.findings,
2578
+ record.round,
2579
+ record.reason,
2580
+ record.sessionFile ?? null,
2581
+ record.requestedAt,
2582
+ );
2583
+ return record;
2584
+ },
2585
+ );
2586
+
2322
2587
  // The label projection outbox (#201). A decided label change is a row until
2323
2588
  // the tracker has applied it; the projector drains in id order, and one
2324
2589
  // issue's ops are atomic as a run — a failed op parks the rest of its issue
@@ -2526,6 +2791,30 @@ export function openStore(dbPath: string): Store {
2526
2791
  runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[] {
2527
2792
  return selectRunsForPr.all(project, prUrl, mergedSinceEpochMs).map(toRecord);
2528
2793
  },
2794
+ createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined {
2795
+ return claimReviewRevision(draft);
2796
+ },
2797
+ pendingReviewRevisions(project: string): ReviewRevisionRecord[] {
2798
+ return selectPendingReviewRevisions.all(project).map(toReviewRevision);
2799
+ },
2800
+ latestReviewRound(project: string, runId: string): number {
2801
+ return selectLatestReviewRound.get(project, runId)?.n ?? 0;
2802
+ },
2803
+ markReviewRevisionDispatched(id: string, at: number): void {
2804
+ markReviewRevisionDispatchedRow.run(at, id);
2805
+ },
2806
+ settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void {
2807
+ settleReviewRevisionRow.run(outcome, at, id);
2808
+ },
2809
+ unsettledReviewRevisions(project: string): ReviewRevisionRecord[] {
2810
+ return selectUnsettledReviewRevisions.all(project).map(toReviewRevision);
2811
+ },
2812
+ requeueReviewRevision(id: string): void {
2813
+ requeueReviewRevisionRow.run(id);
2814
+ },
2815
+ claimRunForReview(runId: string): boolean {
2816
+ return claimRunForReviewRow.run(runId).changes > 0;
2817
+ },
2529
2818
  runsNeedingBaseCheck(project: string, limit = 20): RunRecord[] {
2530
2819
  return selectPendingBaseChecks.all(project, limit).map(toRecord);
2531
2820
  },
@@ -3066,6 +3355,38 @@ export function openStore(dbPath: string): Store {
3066
3355
  return selectUnclassified.all(project, limit).map(toRecord);
3067
3356
  },
3068
3357
 
3358
+ historicalInfraCandidates(
3359
+ project: string,
3360
+ limit = 20,
3361
+ after?: { startedAt: number; rowid: number },
3362
+ ): HistoricalInfraCandidate[] {
3363
+ const cursorAt = after?.startedAt ?? null;
3364
+ const cursorRowid = after?.rowid ?? null;
3365
+ return selectHistoricalInfraCandidates
3366
+ .all(project, cursorAt, cursorAt, cursorAt, cursorRowid, limit)
3367
+ .map((row) => ({ ...toRecord(row), rowid: row.rowid }));
3368
+ },
3369
+
3370
+ historicalInfraCursor(project: string) {
3371
+ const row = selectHistoricalInfraCursor.get(project);
3372
+ return row == null
3373
+ ? undefined
3374
+ : { startedAt: row.cursorStartedAt, rowid: row.cursorRowid, classifierVersion: row.classifierVersion };
3375
+ },
3376
+
3377
+ setHistoricalInfraCursor(
3378
+ project: string,
3379
+ startedAt: number,
3380
+ rowid: number,
3381
+ classifierVersion: string,
3382
+ ): void {
3383
+ upsertHistoricalInfraCursor.run(project, startedAt, rowid, classifierVersion, Date.now());
3384
+ },
3385
+
3386
+ reclassifyInfra(id: string): boolean {
3387
+ return reclassifyToInfra.run(id).changes > 0;
3388
+ },
3389
+
3069
3390
  failureClassCounts(project: string): { cls: FailureClass; n: number }[] {
3070
3391
  return selectFailureClassCounts
3071
3392
  .all(project)