omp-conductor 0.17.0 → 0.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +12 -8
- package/package.json +1 -1
- package/schema/config.schema.json +40 -1
- package/src/admission.ts +263 -44
- package/src/ask.ts +39 -3
- package/src/availability.ts +27 -1
- package/src/backups.ts +2 -2
- package/src/briefs/orchestrator.md +1 -0
- package/src/briefs/worker.md +38 -19
- package/src/command-help.ts +8 -1
- package/src/command-manifest.ts +5 -2
- package/src/commands/arm.ts +6 -3
- package/src/commands/message.ts +32 -4
- package/src/commands/watch.ts +62 -3
- package/src/config-schema.ts +53 -0
- package/src/config.ts +97 -1
- package/src/daemon.ts +1479 -1483
- package/src/decisions.ts +51 -6
- package/src/depends-on.ts +261 -1
- package/src/diff-flags.ts +350 -0
- package/src/digest-schedule.ts +37 -0
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +71 -15
- package/src/fleet.ts +189 -34
- package/src/gitops.ts +103 -24
- package/src/graph-health.ts +20 -7
- package/src/graph.ts +313 -68
- package/src/lifecycle.ts +43 -7
- package/src/omp.ts +42 -0
- package/src/orchestrator-tick.ts +430 -162
- package/src/release-policy.ts +177 -5
- package/src/routing.ts +11 -3
- package/src/session-host.ts +16 -0
- package/src/settlement.ts +1728 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +91 -30
- package/src/setup-wizard.ts +1257 -78
- package/src/setup.ts +153 -6
- package/src/status-render.ts +36 -4
- package/src/store.ts +411 -17
- package/src/tracker/github.ts +607 -12
- package/src/types.ts +331 -5
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +270 -13
- package/src/worker.ts +239 -6
- package/src/worktree.ts +115 -8
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/store.ts
CHANGED
|
@@ -11,7 +11,7 @@
|
|
|
11
11
|
|
|
12
12
|
import { Database } from "bun:sqlite";
|
|
13
13
|
import { randomUUID } from "node:crypto";
|
|
14
|
-
import { existsSync, mkdirSync, rmSync } from "node:fs";
|
|
14
|
+
import { existsSync, mkdirSync, readdirSync, rmSync, unlinkSync } from "node:fs";
|
|
15
15
|
import { dirname, join } from "node:path";
|
|
16
16
|
|
|
17
17
|
import { backupTimestamp, publishTempFile } from "./backups.ts";
|
|
@@ -41,6 +41,7 @@ import type {
|
|
|
41
41
|
FrictionKind,
|
|
42
42
|
FrictionObservation,
|
|
43
43
|
FrictionSignal,
|
|
44
|
+
HistoricalInfraCandidate,
|
|
44
45
|
HeldNotice,
|
|
45
46
|
InterruptCategory,
|
|
46
47
|
HeldNoticeDraft,
|
|
@@ -60,6 +61,9 @@ import type {
|
|
|
60
61
|
ReportKind,
|
|
61
62
|
ReportRecord,
|
|
62
63
|
RecoveryAction,
|
|
64
|
+
ReviewReason,
|
|
65
|
+
ReviewRevisionOutcome,
|
|
66
|
+
ReviewRevisionRecord,
|
|
63
67
|
RunPatch,
|
|
64
68
|
RunRecord,
|
|
65
69
|
RunState,
|
|
@@ -99,6 +103,7 @@ const FRICTION_HOLD_REASONS: ReadonlySet<FrictionAdmissionReason> = new Set([
|
|
|
99
103
|
"failed-attempts",
|
|
100
104
|
"continuations",
|
|
101
105
|
"stale-base",
|
|
106
|
+
"critical-base-verify-error",
|
|
102
107
|
"parent-lookup-error",
|
|
103
108
|
"issue-state-lookup-error",
|
|
104
109
|
"open-pr-lookup-error",
|
|
@@ -141,6 +146,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
|
|
|
141
146
|
autoRetryCount: true,
|
|
142
147
|
autoCompactionCount: true,
|
|
143
148
|
sessionFile: true,
|
|
149
|
+
resumedFromRunId: true,
|
|
144
150
|
prUrl: true,
|
|
145
151
|
headSha: true,
|
|
146
152
|
mergeSha: true,
|
|
@@ -150,6 +156,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
|
|
|
150
156
|
salvageSha: true,
|
|
151
157
|
salvageError: true,
|
|
152
158
|
salvageAckAt: true,
|
|
159
|
+
quarantineDetail: true,
|
|
153
160
|
startedAt: true,
|
|
154
161
|
endedAt: true,
|
|
155
162
|
lastError: true,
|
|
@@ -186,6 +193,7 @@ interface RunRow {
|
|
|
186
193
|
autoRetryCount: number | null;
|
|
187
194
|
autoCompactionCount: number | null;
|
|
188
195
|
sessionFile: string | null;
|
|
196
|
+
resumedFromRunId: string | null;
|
|
189
197
|
prUrl: string | null;
|
|
190
198
|
headSha: string | null;
|
|
191
199
|
mergeSha: string | null;
|
|
@@ -195,6 +203,7 @@ interface RunRow {
|
|
|
195
203
|
salvageSha: string | null;
|
|
196
204
|
salvageError: string | null;
|
|
197
205
|
salvageAckAt: number | null;
|
|
206
|
+
quarantineDetail: string | null;
|
|
198
207
|
startedAt: number;
|
|
199
208
|
endedAt: number | null;
|
|
200
209
|
lastError: string | null;
|
|
@@ -361,6 +370,48 @@ interface MergeLockRow {
|
|
|
361
370
|
at: number;
|
|
362
371
|
}
|
|
363
372
|
|
|
373
|
+
/** The `review_revisions` table exactly as SQLite hands it back (#677). */
|
|
374
|
+
interface ReviewRevisionRow {
|
|
375
|
+
id: string;
|
|
376
|
+
project: string;
|
|
377
|
+
runId: string;
|
|
378
|
+
issue: number;
|
|
379
|
+
prUrl: string;
|
|
380
|
+
headSha: string;
|
|
381
|
+
findings: string;
|
|
382
|
+
round: number;
|
|
383
|
+
reason: string;
|
|
384
|
+
sessionFile: string | null;
|
|
385
|
+
requestedAt: number;
|
|
386
|
+
dispatchedAt: number | null;
|
|
387
|
+
settledAt: number | null;
|
|
388
|
+
outcome: string | null;
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/**
|
|
392
|
+
* NULL columns become absent properties, matching the other row converters:
|
|
393
|
+
* a record read back out of the store deep-equals the one that went in.
|
|
394
|
+
*/
|
|
395
|
+
function toReviewRevision(row: ReviewRevisionRow): ReviewRevisionRecord {
|
|
396
|
+
const record: ReviewRevisionRecord = {
|
|
397
|
+
id: row.id,
|
|
398
|
+
project: row.project,
|
|
399
|
+
runId: row.runId,
|
|
400
|
+
issue: row.issue,
|
|
401
|
+
prUrl: row.prUrl,
|
|
402
|
+
headSha: row.headSha,
|
|
403
|
+
findings: row.findings,
|
|
404
|
+
round: row.round,
|
|
405
|
+
reason: row.reason as ReviewReason,
|
|
406
|
+
requestedAt: row.requestedAt,
|
|
407
|
+
};
|
|
408
|
+
if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
|
|
409
|
+
if (row.dispatchedAt !== null) record.dispatchedAt = row.dispatchedAt;
|
|
410
|
+
if (row.settledAt !== null) record.settledAt = row.settledAt;
|
|
411
|
+
if (row.outcome !== null) record.outcome = row.outcome as ReviewRevisionOutcome;
|
|
412
|
+
return record;
|
|
413
|
+
}
|
|
414
|
+
|
|
364
415
|
/** The `intake_items` table exactly as SQLite hands it back (#299). */
|
|
365
416
|
interface IntakeRow {
|
|
366
417
|
id: string;
|
|
@@ -470,6 +521,7 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
470
521
|
autoRetryCount INTEGER,
|
|
471
522
|
autoCompactionCount INTEGER,
|
|
472
523
|
sessionFile TEXT,
|
|
524
|
+
resumedFromRunId TEXT,
|
|
473
525
|
prUrl TEXT,
|
|
474
526
|
headSha TEXT,
|
|
475
527
|
mergeSha TEXT,
|
|
@@ -492,6 +544,22 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
492
544
|
CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
|
|
493
545
|
CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
|
|
494
546
|
|
|
547
|
+
-- The historical infrastructure reconciliation's per-project review cursor
|
|
548
|
+
-- (#638). A bounded pass examines settled ci-deterministic rows newest-first
|
|
549
|
+
-- and persists the boundary of the last row it definitively decided, so older
|
|
550
|
+
-- repairable rows are reached across later daemon starts instead of being
|
|
551
|
+
-- starved by repeated rescans of the newest non-matches. classifierVersion is
|
|
552
|
+
-- the signature-list fingerprint at the time the cursor advanced; when the
|
|
553
|
+
-- classifier learns a new signature the stamp goes stale and the pass restarts
|
|
554
|
+
-- from the newest row rather than skipping past newly recognised evidence.
|
|
555
|
+
CREATE TABLE IF NOT EXISTS historical_infra_review (
|
|
556
|
+
project TEXT PRIMARY KEY,
|
|
557
|
+
cursorStartedAt INTEGER NOT NULL,
|
|
558
|
+
cursorRowid INTEGER NOT NULL,
|
|
559
|
+
classifierVersion TEXT NOT NULL,
|
|
560
|
+
updatedAt INTEGER NOT NULL
|
|
561
|
+
);
|
|
562
|
+
|
|
495
563
|
CREATE TABLE IF NOT EXISTS base_health (
|
|
496
564
|
project TEXT NOT NULL,
|
|
497
565
|
repo TEXT NOT NULL,
|
|
@@ -693,6 +761,31 @@ CREATE TABLE IF NOT EXISTS merge_locks (
|
|
|
693
761
|
at INTEGER NOT NULL
|
|
694
762
|
);
|
|
695
763
|
|
|
764
|
+
-- One durable review-revision request (#677): an orchestrator returned a
|
|
765
|
+
-- green, run-owned pull request to its worker with blocking findings. Every
|
|
766
|
+
-- field the daemon needs to resume the exact session is persisted BEFORE the
|
|
767
|
+
-- worker is woken -- the findings, the exact reviewed head, the round number,
|
|
768
|
+
-- and the target run/session -- and the row doubles as the duplicate guard: a
|
|
769
|
+
-- revision whose row exists pending is one nobody may queue again.
|
|
770
|
+
CREATE TABLE IF NOT EXISTS review_revisions (
|
|
771
|
+
id TEXT PRIMARY KEY,
|
|
772
|
+
project TEXT NOT NULL,
|
|
773
|
+
runId TEXT NOT NULL,
|
|
774
|
+
issue INTEGER NOT NULL,
|
|
775
|
+
prUrl TEXT NOT NULL,
|
|
776
|
+
headSha TEXT NOT NULL,
|
|
777
|
+
findings TEXT NOT NULL,
|
|
778
|
+
round INTEGER NOT NULL,
|
|
779
|
+
reason TEXT NOT NULL,
|
|
780
|
+
sessionFile TEXT,
|
|
781
|
+
requestedAt INTEGER NOT NULL,
|
|
782
|
+
dispatchedAt INTEGER,
|
|
783
|
+
settledAt INTEGER,
|
|
784
|
+
outcome TEXT
|
|
785
|
+
);
|
|
786
|
+
CREATE INDEX IF NOT EXISTS review_revisions_pending
|
|
787
|
+
ON review_revisions (project, dispatchedAt, runId);
|
|
788
|
+
|
|
696
789
|
-- Questions the orchestrator has put to its operator, and their answers (#136).
|
|
697
790
|
--
|
|
698
791
|
-- The reason this is a table and not the session's memory: a question lived only
|
|
@@ -903,6 +996,7 @@ function toRecord(row: RunRow): RunRecord {
|
|
|
903
996
|
startedAt: row.startedAt,
|
|
904
997
|
};
|
|
905
998
|
if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
|
|
999
|
+
if (row.resumedFromRunId !== null) record.resumedFromRunId = row.resumedFromRunId;
|
|
906
1000
|
if (row.prUrl !== null) record.prUrl = row.prUrl;
|
|
907
1001
|
if (row.headSha !== null) record.headSha = row.headSha;
|
|
908
1002
|
if (row.mergeSha !== null) record.mergeSha = row.mergeSha;
|
|
@@ -912,6 +1006,7 @@ function toRecord(row: RunRow): RunRecord {
|
|
|
912
1006
|
if (row.salvageSha !== null) record.salvageSha = row.salvageSha;
|
|
913
1007
|
if (row.salvageError !== null) record.salvageError = row.salvageError;
|
|
914
1008
|
if (row.salvageAckAt !== null) record.salvageAckAt = row.salvageAckAt;
|
|
1009
|
+
if (row.quarantineDetail !== null) record.quarantineDetail = row.quarantineDetail;
|
|
915
1010
|
if (row.endedAt !== null) record.endedAt = row.endedAt;
|
|
916
1011
|
if (row.lastError !== null) record.lastError = row.lastError;
|
|
917
1012
|
if (row.settlementFlags !== null) {
|
|
@@ -1143,12 +1238,59 @@ export function dbPath(): string {
|
|
|
1143
1238
|
return join(stateDir(), "conductor.db");
|
|
1144
1239
|
}
|
|
1145
1240
|
|
|
1241
|
+
/**
|
|
1242
|
+
* Live worker count across the run store, opening the database WITHOUT
|
|
1243
|
+
* creating or upgrading it. The setup apply's quiescence preflight must count
|
|
1244
|
+
* workers before the acknowledged barrier, and database preparation is itself
|
|
1245
|
+
* one of the mutations that barrier must precede — so the observation that
|
|
1246
|
+
* gates it cannot be the thing that creates or migrates the store (#618). The
|
|
1247
|
+
* on-disk database is opened read-only (never created, never "prepared"); a
|
|
1248
|
+
* missing file reports zero because no run has ever been persisted, which the
|
|
1249
|
+
* caller only trusts when no daemon is running either. An unreadable or
|
|
1250
|
+
* schema-broken file throws rather than reading as zero — fail closed, never
|
|
1251
|
+
* absence.
|
|
1252
|
+
*/
|
|
1253
|
+
export function liveWorkersReadOnly(project?: string): number {
|
|
1254
|
+
const path = dbPath();
|
|
1255
|
+
if (!existsSync(path)) return 0;
|
|
1256
|
+
let db: Database | undefined;
|
|
1257
|
+
try {
|
|
1258
|
+
db = new Database(path, { create: false, readonly: true });
|
|
1259
|
+
db.exec("PRAGMA busy_timeout = 5000;");
|
|
1260
|
+
const placeholders = LIVE_STATES.map(() => "?").join(", ");
|
|
1261
|
+
// `undefined` is the host-global selector: every project's live rows count,
|
|
1262
|
+
// never only the rows of a project literally named "" — a host-global
|
|
1263
|
+
// observation that scoped to `project = ''` hid every real named row and
|
|
1264
|
+
// read zero beside a running fleet (#651 review #3).
|
|
1265
|
+
const row =
|
|
1266
|
+
project === undefined
|
|
1267
|
+
? db
|
|
1268
|
+
.query<{ n: number }, SqlValue[]>(
|
|
1269
|
+
`SELECT COUNT(*) AS n FROM runs WHERE state IN (${placeholders})`,
|
|
1270
|
+
)
|
|
1271
|
+
.get(...LIVE_STATES)
|
|
1272
|
+
: db
|
|
1273
|
+
.query<{ n: number }, SqlValue[]>(
|
|
1274
|
+
`SELECT COUNT(*) AS n FROM runs WHERE project = ? AND state IN (${placeholders})`,
|
|
1275
|
+
)
|
|
1276
|
+
.get(project, ...LIVE_STATES);
|
|
1277
|
+
return row?.n ?? 0;
|
|
1278
|
+
} catch (err) {
|
|
1279
|
+
throw new Error(
|
|
1280
|
+
`cannot read ${path} to prove the fleet is quiescent: ${err instanceof Error ? err.message : String(err)}`,
|
|
1281
|
+
);
|
|
1282
|
+
} finally {
|
|
1283
|
+
db?.close();
|
|
1284
|
+
}
|
|
1285
|
+
}
|
|
1286
|
+
|
|
1146
1287
|
/** The stem every restorable `conductor.db` snapshot is published under. */
|
|
1147
1288
|
export const DB_SNAPSHOT_STEM = "conductor.db.bak";
|
|
1148
1289
|
|
|
1149
1290
|
/**
|
|
1150
|
-
* A
|
|
1151
|
-
*
|
|
1291
|
+
* A single-file, transaction-consistent copy of the store at `source` into
|
|
1292
|
+
* `target` — the shared core of {@link snapshotDb} and the setup apply's
|
|
1293
|
+
* rollback capture.
|
|
1152
1294
|
*
|
|
1153
1295
|
* Uses SQLite's own `VACUUM INTO`, never a plain file copy: the store opens
|
|
1154
1296
|
* with `journal_mode=WAL`, and a `copyFileSync` can capture a torn database
|
|
@@ -1158,34 +1300,79 @@ export const DB_SNAPSHOT_STEM = "conductor.db.bak";
|
|
|
1158
1300
|
* writes a single-file snapshot, so the result opens cleanly in a fresh
|
|
1159
1301
|
* connection and contains every committed row.
|
|
1160
1302
|
*
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1163
|
-
*
|
|
1303
|
+
* Runs against its own read of the source, deliberately not the caller's
|
|
1304
|
+
* writer connection, so a snapshot can be taken while a writer holds an
|
|
1305
|
+
* uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
|
|
1164
1306
|
* refused.
|
|
1165
|
-
*
|
|
1166
|
-
* Returns the published snapshot path.
|
|
1167
1307
|
*/
|
|
1168
|
-
export function
|
|
1308
|
+
export function vacuumInto(source: string, target: string): void {
|
|
1169
1309
|
if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
|
|
1170
1310
|
if (!existsSync(source)) {
|
|
1171
1311
|
throw new Error(`cannot snapshot ${source}: the store does not exist`);
|
|
1172
1312
|
}
|
|
1173
|
-
mkdirSync(destDir, { recursive: true });
|
|
1174
|
-
const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
|
|
1175
1313
|
let connection: Database | undefined;
|
|
1176
1314
|
try {
|
|
1177
1315
|
connection = new Database(source);
|
|
1178
1316
|
connection.exec("PRAGMA busy_timeout = 5000;");
|
|
1179
1317
|
// `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
|
|
1180
|
-
connection.exec(`VACUUM INTO '${
|
|
1318
|
+
connection.exec(`VACUUM INTO '${target.replaceAll("'", "''")}'`);
|
|
1319
|
+
} finally {
|
|
1320
|
+
connection?.close();
|
|
1321
|
+
}
|
|
1322
|
+
}
|
|
1323
|
+
|
|
1324
|
+
/**
|
|
1325
|
+
* A fresh, restorable copy of the store at `source`, published under `destDir`
|
|
1326
|
+
* with the same temp-then-atomic-publish convention as the config backups.
|
|
1327
|
+
*
|
|
1328
|
+
* The stem carries `at` (default now) so the filename's timestamp is the
|
|
1329
|
+
* snapshot's logical instant — the cadence passes its tick time, keeping the
|
|
1330
|
+
* name, the day marker and the retention ordering in one timeline.
|
|
1331
|
+
*
|
|
1332
|
+
* Returns the published snapshot path.
|
|
1333
|
+
*/
|
|
1334
|
+
export function snapshotDb(source: string, destDir: string, at: number = Date.now()): string {
|
|
1335
|
+
if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
|
|
1336
|
+
if (!existsSync(source)) {
|
|
1337
|
+
throw new Error(`cannot snapshot ${source}: the store does not exist`);
|
|
1338
|
+
}
|
|
1339
|
+
mkdirSync(destDir, { recursive: true });
|
|
1340
|
+
const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
|
|
1341
|
+
try {
|
|
1342
|
+
vacuumInto(source, temporary);
|
|
1181
1343
|
} catch (err) {
|
|
1182
1344
|
// Never leave a half-written snapshot on disk.
|
|
1183
1345
|
rmSync(temporary, { force: true });
|
|
1184
1346
|
throw err;
|
|
1185
|
-
} finally {
|
|
1186
|
-
connection?.close();
|
|
1187
1347
|
}
|
|
1188
|
-
return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp()}`);
|
|
1348
|
+
return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp(at)}`);
|
|
1349
|
+
}
|
|
1350
|
+
|
|
1351
|
+
/**
|
|
1352
|
+
* How many conductor.db snapshots the daemon's daily cadence keeps — a week
|
|
1353
|
+
* of ledger history at one snapshot per day. Deliberately a fixed count, not
|
|
1354
|
+
* an age or space policy: the store's ledger is append-only and the issue
|
|
1355
|
+
* forbids inventing a fleet-wide retention policy, so pruning is
|
|
1356
|
+
* configured-count only (#289).
|
|
1357
|
+
*/
|
|
1358
|
+
export const DB_SNAPSHOT_RETENTION = 7;
|
|
1359
|
+
|
|
1360
|
+
/**
|
|
1361
|
+
* Prune a snapshot directory to the newest `keep` conductor.db snapshots.
|
|
1362
|
+
*
|
|
1363
|
+
* Snapshot stems embed the publish timestamp (`YYYY-MM-DDTHH-MM-SS…`), so
|
|
1364
|
+
* lexicographic order is chronological; only files named with the shared
|
|
1365
|
+
* `DB_SNAPSHOT_STEM` prefix are considered, never unrelated backups next to
|
|
1366
|
+
* them. Idempotent: a directory already at or under the bound removes
|
|
1367
|
+
* nothing. Returns the number of files removed.
|
|
1368
|
+
*/
|
|
1369
|
+
export function pruneDbSnapshots(dir: string, keep: number): number {
|
|
1370
|
+
const names = readdirSync(dir)
|
|
1371
|
+
.filter((name) => name.startsWith(DB_SNAPSHOT_STEM))
|
|
1372
|
+
.sort();
|
|
1373
|
+
const stale = names.length > keep ? names.slice(0, names.length - keep) : [];
|
|
1374
|
+
for (const name of stale) unlinkSync(join(dir, name));
|
|
1375
|
+
return stale.length;
|
|
1189
1376
|
}
|
|
1190
1377
|
|
|
1191
1378
|
/** The UTC calendar day (`YYYY-MM-DD`) a moment falls on — the partition key
|
|
@@ -1272,6 +1459,14 @@ export function openStore(dbPath: string): Store {
|
|
|
1272
1459
|
db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
|
|
1273
1460
|
}
|
|
1274
1461
|
}
|
|
1462
|
+
// Rows written before #737 were never asked whether their tree was
|
|
1463
|
+
// quarantined, and a NULL means exactly that: the pre-quarantine passes
|
|
1464
|
+
// reaped the same trees they always had, so nothing before this release
|
|
1465
|
+
// needs backfilling — the column starts null and only a post-release pass
|
|
1466
|
+
// that actually refuses a broken store writes it.
|
|
1467
|
+
if (!columns.some((column) => column.name === "quarantineDetail")) {
|
|
1468
|
+
db.exec("ALTER TABLE runs ADD COLUMN quarantineDetail TEXT");
|
|
1469
|
+
}
|
|
1275
1470
|
// Rows written before the settlement audit existed (#128) were never audited,
|
|
1276
1471
|
// so a NULL here means "no audit ran" and reads the same as "found nothing".
|
|
1277
1472
|
// That conflation is deliberate and harmless: the flags are advisory, and no
|
|
@@ -1306,6 +1501,13 @@ export function openStore(dbPath: string): Store {
|
|
|
1306
1501
|
if (!columns.some((column) => column.name === "model")) {
|
|
1307
1502
|
db.exec("ALTER TABLE runs ADD COLUMN model TEXT");
|
|
1308
1503
|
}
|
|
1504
|
+
// Resume provenance (#567): rows written before #564 landed were never
|
|
1505
|
+
// resumed, so NULL is the honest reading of "no resume happened" — never
|
|
1506
|
+
// "resumed from unknown". No backfill: the identity existed only in the
|
|
1507
|
+
// dispatch decision, and deriving it now would guess at history.
|
|
1508
|
+
if (!columns.some((column) => column.name === "resumedFromRunId")) {
|
|
1509
|
+
db.exec("ALTER TABLE runs ADD COLUMN resumedFromRunId TEXT");
|
|
1510
|
+
}
|
|
1309
1511
|
// The count of in-session provider 429s a run recorded was first written by
|
|
1310
1512
|
// the worker (#573). Rows predating the column are NULL, which is the honest
|
|
1311
1513
|
// reading: the count was not recorded, so the classifier must not treat the
|
|
@@ -1465,10 +1667,10 @@ export function openStore(dbPath: string): Store {
|
|
|
1465
1667
|
const insertRun = db.query<unknown, SqlValue[]>(
|
|
1466
1668
|
`INSERT INTO runs (
|
|
1467
1669
|
id, project, issue, repo, branch, worktree, state, attempt, turns,
|
|
1468
|
-
maxTurns, spendUsd, sessionFile, prUrl, headSha, mergeSha, baseRef,
|
|
1670
|
+
maxTurns, spendUsd, sessionFile, resumedFromRunId, prUrl, headSha, mergeSha, baseRef,
|
|
1469
1671
|
baseCheck, baseCheckAt, salvageSha, salvageError, salvageAckAt, startedAt,
|
|
1470
1672
|
endedAt, lastError, settlementFlags, report
|
|
1471
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
1673
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
1472
1674
|
);
|
|
1473
1675
|
const selectRun = db.query<RunRow, [string]>(`SELECT * FROM runs WHERE id = ?`);
|
|
1474
1676
|
const selectActive = db.query<RunRow, SqlValue[]>(
|
|
@@ -1644,6 +1846,56 @@ export function openStore(dbPath: string): Store {
|
|
|
1644
1846
|
ORDER BY startedAt DESC, rowid DESC
|
|
1645
1847
|
LIMIT ?`,
|
|
1646
1848
|
);
|
|
1849
|
+
// The settled rows a classifier correction can no longer reach (#638). A
|
|
1850
|
+
// `ci-deterministic`/`escalate` row never re-enters `selectUnclassified`
|
|
1851
|
+
// (its class is set and its recovery is not retryable), so evidence the
|
|
1852
|
+
// classifier learned after the row was written — the codeload setup 429 of
|
|
1853
|
+
// #177 — stays a charged attempt forever. This WHERE is the historical
|
|
1854
|
+
// reconciliation's exact predicate: nothing but rows currently reading
|
|
1855
|
+
// `ci-deterministic`, never a class a human re-read, and the class itself is
|
|
1856
|
+
// only ever produced for `state = 'failed'`. Bounded like every other sweep
|
|
1857
|
+
// read, because each candidate costs tracker calls to re-fetch its evidence.
|
|
1858
|
+
// `after` is the persisted review cursor: candidates are only rows strictly
|
|
1859
|
+
// older than it, so a bounded pass progresses through a long misclassified
|
|
1860
|
+
// history instead of rescanning the newest non-matches forever. A NULL cursor
|
|
1861
|
+
// (first pass, or one rendered stale by a classifier-signature change) means
|
|
1862
|
+
// "start from the newest". `SELECT rowid, *` exposes the hidden tie-breaker
|
|
1863
|
+
// so the pass can record an exact resume boundary.
|
|
1864
|
+
const selectHistoricalInfraCandidates = db.query<RunRow & { rowid: number }, SqlValue[]>(
|
|
1865
|
+
`SELECT rowid, * FROM runs
|
|
1866
|
+
WHERE project = ? AND failureClass = 'ci-deterministic' AND state = 'failed'
|
|
1867
|
+
AND (? IS NULL OR (startedAt < ? OR (startedAt = ? AND rowid < ?)))
|
|
1868
|
+
ORDER BY startedAt DESC, rowid DESC
|
|
1869
|
+
LIMIT ?`,
|
|
1870
|
+
);
|
|
1871
|
+
const selectHistoricalInfraCursor = db.query<
|
|
1872
|
+
{ cursorStartedAt: number; cursorRowid: number; classifierVersion: string },
|
|
1873
|
+
[string]
|
|
1874
|
+
>(
|
|
1875
|
+
`SELECT cursorStartedAt, cursorRowid, classifierVersion FROM historical_infra_review
|
|
1876
|
+
WHERE project = ?`,
|
|
1877
|
+
);
|
|
1878
|
+
const upsertHistoricalInfraCursor = db.query<
|
|
1879
|
+
unknown,
|
|
1880
|
+
[string, number, number, string, number]
|
|
1881
|
+
>(
|
|
1882
|
+
`INSERT INTO historical_infra_review
|
|
1883
|
+
(project, cursorStartedAt, cursorRowid, classifierVersion, updatedAt)
|
|
1884
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1885
|
+
ON CONFLICT(project) DO UPDATE SET
|
|
1886
|
+
cursorStartedAt = excluded.cursorStartedAt,
|
|
1887
|
+
cursorRowid = excluded.cursorRowid,
|
|
1888
|
+
classifierVersion = excluded.classifierVersion,
|
|
1889
|
+
updatedAt = excluded.updatedAt`,
|
|
1890
|
+
);
|
|
1891
|
+
// The one mutation the historical reconciliation may make. Guarded by the
|
|
1892
|
+
// current class, so a second pass is a no-op and a race can never reclassify
|
|
1893
|
+
// a row a human has re-read. Deliberately only `failureClass` — never
|
|
1894
|
+
// `recoveryAction` or `recoveredAt`: relieving the budget is the point,
|
|
1895
|
+
// re-animating a months-old run into the requeue sweep is not (#638).
|
|
1896
|
+
const reclassifyToInfra = db.query<unknown, [string]>(
|
|
1897
|
+
`UPDATE runs SET failureClass = 'ci-infra' WHERE id = ? AND failureClass = 'ci-deterministic'`,
|
|
1898
|
+
);
|
|
1647
1899
|
const selectFailureClassCounts = db.query<{ cls: string; n: number }, [string]>(
|
|
1648
1900
|
`SELECT failureClass AS cls, COUNT(*) AS n FROM runs
|
|
1649
1901
|
WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
|
|
@@ -1671,6 +1923,21 @@ export function openStore(dbPath: string): Store {
|
|
|
1671
1923
|
AND (salvageSha IS NOT NULL OR salvageError IS NOT NULL)
|
|
1672
1924
|
ORDER BY issue ASC`,
|
|
1673
1925
|
);
|
|
1926
|
+
// Quarantined retained trees (#737), newest attempt per issue — the same
|
|
1927
|
+
// shape as `selectSalvaged` because the same change can land twice: a row
|
|
1928
|
+
// long since reaped must not keep its issue looking quarantined, and an
|
|
1929
|
+
// older quarantined attempt must not claim the issue while a newer,
|
|
1930
|
+
// healthy one replaced it.
|
|
1931
|
+
const selectQuarantined = db.query<RunRow, [string]>(
|
|
1932
|
+
`SELECT * FROM runs
|
|
1933
|
+
WHERE rowid IN (
|
|
1934
|
+
SELECT MAX(rowid) FROM runs
|
|
1935
|
+
WHERE project = ?
|
|
1936
|
+
GROUP BY issue
|
|
1937
|
+
)
|
|
1938
|
+
AND quarantineDetail IS NOT NULL
|
|
1939
|
+
ORDER BY issue ASC`,
|
|
1940
|
+
);
|
|
1674
1941
|
// Newest attempt for one issue. `startedAt` is millisecond-resolution and two
|
|
1675
1942
|
// attempts could in principle share one, so rowid breaks the tie by insertion
|
|
1676
1943
|
// order — a `tail` that attached to the older of two same-millisecond attempts
|
|
@@ -2319,6 +2586,72 @@ export function openStore(dbPath: string): Store {
|
|
|
2319
2586
|
},
|
|
2320
2587
|
);
|
|
2321
2588
|
|
|
2589
|
+
// The durable review-revision request outbox (#677): the orchestrator's verb
|
|
2590
|
+
// writes the row (findings, head, round, target session) and the daemon's
|
|
2591
|
+
// next dispatch pass wakes it. The duplicate in-flight guard is the insert:
|
|
2592
|
+
// a pending row for the same run refuses the second request atomically, and
|
|
2593
|
+
// once dispatched the run row's own state (pushed-green → running) is the
|
|
2594
|
+
// in-flight marker.
|
|
2595
|
+
const insertReviewRevision = db.query<unknown, SqlValue[]>(
|
|
2596
|
+
`INSERT INTO review_revisions
|
|
2597
|
+
(id, project, runId, issue, prUrl, headSha, findings, round, reason, sessionFile, requestedAt, dispatchedAt, settledAt, outcome)
|
|
2598
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL)`,
|
|
2599
|
+
);
|
|
2600
|
+
const selectPendingReviewRevisions = db.query<ReviewRevisionRow, [string]>(
|
|
2601
|
+
`SELECT * FROM review_revisions
|
|
2602
|
+
WHERE project = ? AND dispatchedAt IS NULL AND settledAt IS NULL
|
|
2603
|
+
ORDER BY requestedAt ASC, rowid ASC`,
|
|
2604
|
+
);
|
|
2605
|
+
const selectPendingReviewForRun = db.query<ReviewRevisionRow, [string, string]>(
|
|
2606
|
+
`SELECT * FROM review_revisions
|
|
2607
|
+
WHERE project = ? AND runId = ? AND dispatchedAt IS NULL AND settledAt IS NULL
|
|
2608
|
+
LIMIT 1`,
|
|
2609
|
+
);
|
|
2610
|
+
const selectLatestReviewRound = db.query<{ n: number | null }, [string, string]>(
|
|
2611
|
+
`SELECT MAX(round) AS n FROM review_revisions WHERE project = ? AND runId = ?`,
|
|
2612
|
+
);
|
|
2613
|
+
const markReviewRevisionDispatchedRow = db.query<unknown, [number, string]>(
|
|
2614
|
+
`UPDATE review_revisions SET dispatchedAt = ? WHERE id = ?`,
|
|
2615
|
+
);
|
|
2616
|
+
const settleReviewRevisionRow = db.query<unknown, [string, number, string]>(
|
|
2617
|
+
`UPDATE review_revisions SET outcome = ?, settledAt = ? WHERE id = ?`,
|
|
2618
|
+
);
|
|
2619
|
+
const selectUnsettledReviewRevisions = db.query<ReviewRevisionRow, [string]>(
|
|
2620
|
+
`SELECT * FROM review_revisions
|
|
2621
|
+
WHERE project = ? AND settledAt IS NULL
|
|
2622
|
+
ORDER BY requestedAt ASC, rowid ASC`,
|
|
2623
|
+
);
|
|
2624
|
+
const requeueReviewRevisionRow = db.query<unknown, [string]>(
|
|
2625
|
+
`UPDATE review_revisions SET dispatchedAt = NULL WHERE id = ?`,
|
|
2626
|
+
);
|
|
2627
|
+
const claimRunForReviewRow = db.query<unknown, [string]>(
|
|
2628
|
+
`UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'pushed-green'`,
|
|
2629
|
+
);
|
|
2630
|
+
|
|
2631
|
+
// The claim and the insert share a transaction so two concurrent revision
|
|
2632
|
+
// requests for one run cannot both win: the second sees the first's pending
|
|
2633
|
+
// row and returns undefined, which the verb turns into the in-flight refusal.
|
|
2634
|
+
const claimReviewRevision = db.transaction(
|
|
2635
|
+
(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined => {
|
|
2636
|
+
if (selectPendingReviewForRun.get(draft.project, draft.runId) !== null) return undefined;
|
|
2637
|
+
const record: ReviewRevisionRecord = { ...draft, id: crypto.randomUUID() };
|
|
2638
|
+
insertReviewRevision.run(
|
|
2639
|
+
record.id,
|
|
2640
|
+
record.project,
|
|
2641
|
+
record.runId,
|
|
2642
|
+
record.issue,
|
|
2643
|
+
record.prUrl,
|
|
2644
|
+
record.headSha,
|
|
2645
|
+
record.findings,
|
|
2646
|
+
record.round,
|
|
2647
|
+
record.reason,
|
|
2648
|
+
record.sessionFile ?? null,
|
|
2649
|
+
record.requestedAt,
|
|
2650
|
+
);
|
|
2651
|
+
return record;
|
|
2652
|
+
},
|
|
2653
|
+
);
|
|
2654
|
+
|
|
2322
2655
|
// The label projection outbox (#201). A decided label change is a row until
|
|
2323
2656
|
// the tracker has applied it; the projector drains in id order, and one
|
|
2324
2657
|
// issue's ops are atomic as a run — a failed op parks the rest of its issue
|
|
@@ -2443,6 +2776,7 @@ export function openStore(dbPath: string): Store {
|
|
|
2443
2776
|
record.maxTurns,
|
|
2444
2777
|
record.spendUsd,
|
|
2445
2778
|
toSql(record.sessionFile),
|
|
2779
|
+
toSql(record.resumedFromRunId),
|
|
2446
2780
|
toSql(record.prUrl),
|
|
2447
2781
|
toSql(record.headSha),
|
|
2448
2782
|
toSql(record.mergeSha),
|
|
@@ -2526,6 +2860,30 @@ export function openStore(dbPath: string): Store {
|
|
|
2526
2860
|
runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[] {
|
|
2527
2861
|
return selectRunsForPr.all(project, prUrl, mergedSinceEpochMs).map(toRecord);
|
|
2528
2862
|
},
|
|
2863
|
+
createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined {
|
|
2864
|
+
return claimReviewRevision(draft);
|
|
2865
|
+
},
|
|
2866
|
+
pendingReviewRevisions(project: string): ReviewRevisionRecord[] {
|
|
2867
|
+
return selectPendingReviewRevisions.all(project).map(toReviewRevision);
|
|
2868
|
+
},
|
|
2869
|
+
latestReviewRound(project: string, runId: string): number {
|
|
2870
|
+
return selectLatestReviewRound.get(project, runId)?.n ?? 0;
|
|
2871
|
+
},
|
|
2872
|
+
markReviewRevisionDispatched(id: string, at: number): void {
|
|
2873
|
+
markReviewRevisionDispatchedRow.run(at, id);
|
|
2874
|
+
},
|
|
2875
|
+
settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void {
|
|
2876
|
+
settleReviewRevisionRow.run(outcome, at, id);
|
|
2877
|
+
},
|
|
2878
|
+
unsettledReviewRevisions(project: string): ReviewRevisionRecord[] {
|
|
2879
|
+
return selectUnsettledReviewRevisions.all(project).map(toReviewRevision);
|
|
2880
|
+
},
|
|
2881
|
+
requeueReviewRevision(id: string): void {
|
|
2882
|
+
requeueReviewRevisionRow.run(id);
|
|
2883
|
+
},
|
|
2884
|
+
claimRunForReview(runId: string): boolean {
|
|
2885
|
+
return claimRunForReviewRow.run(runId).changes > 0;
|
|
2886
|
+
},
|
|
2529
2887
|
runsNeedingBaseCheck(project: string, limit = 20): RunRecord[] {
|
|
2530
2888
|
return selectPendingBaseChecks.all(project, limit).map(toRecord);
|
|
2531
2889
|
},
|
|
@@ -2596,6 +2954,10 @@ export function openStore(dbPath: string): Store {
|
|
|
2596
2954
|
return selectSalvaged.all(project).map(toRecord);
|
|
2597
2955
|
},
|
|
2598
2956
|
|
|
2957
|
+
quarantinedRuns(project: string): RunRecord[] {
|
|
2958
|
+
return selectQuarantined.all(project).map(toRecord);
|
|
2959
|
+
},
|
|
2960
|
+
|
|
2599
2961
|
attemptsFor(project: string, issue: number): number {
|
|
2600
2962
|
return countAttempts.get(project, issue)?.n ?? 0;
|
|
2601
2963
|
},
|
|
@@ -3066,6 +3428,38 @@ export function openStore(dbPath: string): Store {
|
|
|
3066
3428
|
return selectUnclassified.all(project, limit).map(toRecord);
|
|
3067
3429
|
},
|
|
3068
3430
|
|
|
3431
|
+
historicalInfraCandidates(
|
|
3432
|
+
project: string,
|
|
3433
|
+
limit = 20,
|
|
3434
|
+
after?: { startedAt: number; rowid: number },
|
|
3435
|
+
): HistoricalInfraCandidate[] {
|
|
3436
|
+
const cursorAt = after?.startedAt ?? null;
|
|
3437
|
+
const cursorRowid = after?.rowid ?? null;
|
|
3438
|
+
return selectHistoricalInfraCandidates
|
|
3439
|
+
.all(project, cursorAt, cursorAt, cursorAt, cursorRowid, limit)
|
|
3440
|
+
.map((row) => ({ ...toRecord(row), rowid: row.rowid }));
|
|
3441
|
+
},
|
|
3442
|
+
|
|
3443
|
+
historicalInfraCursor(project: string) {
|
|
3444
|
+
const row = selectHistoricalInfraCursor.get(project);
|
|
3445
|
+
return row == null
|
|
3446
|
+
? undefined
|
|
3447
|
+
: { startedAt: row.cursorStartedAt, rowid: row.cursorRowid, classifierVersion: row.classifierVersion };
|
|
3448
|
+
},
|
|
3449
|
+
|
|
3450
|
+
setHistoricalInfraCursor(
|
|
3451
|
+
project: string,
|
|
3452
|
+
startedAt: number,
|
|
3453
|
+
rowid: number,
|
|
3454
|
+
classifierVersion: string,
|
|
3455
|
+
): void {
|
|
3456
|
+
upsertHistoricalInfraCursor.run(project, startedAt, rowid, classifierVersion, Date.now());
|
|
3457
|
+
},
|
|
3458
|
+
|
|
3459
|
+
reclassifyInfra(id: string): boolean {
|
|
3460
|
+
return reclassifyToInfra.run(id).changes > 0;
|
|
3461
|
+
},
|
|
3462
|
+
|
|
3069
3463
|
failureClassCounts(project: string): { cls: FailureClass; n: number }[] {
|
|
3070
3464
|
return selectFailureClassCounts
|
|
3071
3465
|
.all(project)
|