omp-conductor 0.15.13 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/REFERENCE.md +72 -2
- package/package.json +2 -1
- package/schema/config.schema.json +6 -0
- package/src/admission.ts +745 -0
- package/src/ask.ts +47 -0
- package/src/backups.ts +19 -7
- package/src/board.ts +1 -2
- package/src/briefs/orchestrator.md +62 -4
- package/src/cli.ts +26 -0
- package/src/commands/context.ts +3 -0
- package/src/commands/decision.ts +10 -1
- package/src/commands/doctor.ts +2 -0
- package/src/commands/message.ts +8 -1
- package/src/commands/restart.ts +15 -3
- package/src/commands/restore-db.ts +146 -0
- package/src/commands/stop.ts +24 -15
- package/src/commands/unfreeze.ts +56 -0
- package/src/commands/watch.ts +77 -0
- package/src/config-schema.ts +9 -0
- package/src/config.ts +24 -0
- package/src/daemon.ts +239 -530
- package/src/dashboard/server.ts +2 -1
- package/src/decisions.ts +32 -7
- package/src/depends-on.ts +73 -0
- package/src/doctor.ts +178 -5
- package/src/escalate.ts +114 -15
- package/src/failure-class.ts +47 -0
- package/src/fleet.ts +41 -410
- package/src/gitops.ts +86 -1
- package/src/log.ts +40 -0
- package/src/model-fallback.ts +3 -2
- package/src/omp-settings.ts +114 -0
- package/src/omp.ts +39 -0
- package/src/orchestrator-tick.ts +7 -1
- package/src/reports.ts +124 -12
- package/src/session-host.ts +6 -0
- package/src/setup-wizard.ts +36 -0
- package/src/setup.ts +58 -1
- package/src/status-render.ts +445 -0
- package/src/stop-provenance.ts +53 -0
- package/src/store.ts +352 -11
- package/src/types.ts +187 -4
- package/src/unblock.ts +1 -1
- package/src/upgrade-verify.ts +1 -1
- package/src/upgrade.ts +1 -2
- package/src/verbs/server.ts +25 -0
- package/src/worker.ts +162 -10
package/src/store.ts
CHANGED
|
@@ -10,13 +10,24 @@
|
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
12
|
import { Database } from "bun:sqlite";
|
|
13
|
-
import {
|
|
13
|
+
import { randomUUID } from "node:crypto";
|
|
14
|
+
import { existsSync, mkdirSync, rmSync } from "node:fs";
|
|
14
15
|
import { dirname, join } from "node:path";
|
|
15
16
|
|
|
17
|
+
import { backupTimestamp, publishTempFile } from "./backups.ts";
|
|
16
18
|
import { stateDir } from "./config.ts";
|
|
17
19
|
import { DECISION_TTL_MS, DEFAULT_CAPS, DIGEST_BACKLOG_LIMIT } from "./types.ts";
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* `expiresAt` for a watch (#459). It has no human deadline — a condition that
|
|
23
|
+
* simply has not fired yet is a different thing from a question nobody
|
|
24
|
+
* answered, and silently expiring it could drop a release. Far enough ahead
|
|
25
|
+
* that `expireDueDecisions` (`expiresAt <= now`) never closes it.
|
|
26
|
+
*/
|
|
27
|
+
const DECISION_WATCH_NEVER_EXPIRES = Number.MAX_SAFE_INTEGER;
|
|
18
28
|
import { dispatchInfra, neverStarted } from "./failure-class.ts";
|
|
19
29
|
import type {
|
|
30
|
+
BaseFreeze,
|
|
20
31
|
BaseHealth,
|
|
21
32
|
DecisionDraft,
|
|
22
33
|
FailureClass,
|
|
@@ -121,6 +132,14 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
|
|
|
121
132
|
turns: true,
|
|
122
133
|
maxTurns: true,
|
|
123
134
|
spendUsd: true,
|
|
135
|
+
provider429Count: true,
|
|
136
|
+
resolvedModel: true,
|
|
137
|
+
resolvedProvider: true,
|
|
138
|
+
retryFallbacks: true,
|
|
139
|
+
retryFallbackSucceeded: true,
|
|
140
|
+
modelRecoveries: true,
|
|
141
|
+
autoRetryCount: true,
|
|
142
|
+
autoCompactionCount: true,
|
|
124
143
|
sessionFile: true,
|
|
125
144
|
prUrl: true,
|
|
126
145
|
headSha: true,
|
|
@@ -158,6 +177,14 @@ interface RunRow {
|
|
|
158
177
|
turns: number;
|
|
159
178
|
maxTurns: number;
|
|
160
179
|
spendUsd: number;
|
|
180
|
+
provider429Count: number | null;
|
|
181
|
+
resolvedModel: string | null;
|
|
182
|
+
resolvedProvider: string | null;
|
|
183
|
+
retryFallbacks: string | null;
|
|
184
|
+
retryFallbackSucceeded: number | null;
|
|
185
|
+
modelRecoveries: number | null;
|
|
186
|
+
autoRetryCount: number | null;
|
|
187
|
+
autoCompactionCount: number | null;
|
|
161
188
|
sessionFile: string | null;
|
|
162
189
|
prUrl: string | null;
|
|
163
190
|
headSha: string | null;
|
|
@@ -204,6 +231,33 @@ function toBaseHealth(row: BaseHealthRow): BaseHealth {
|
|
|
204
231
|
return health;
|
|
205
232
|
}
|
|
206
233
|
|
|
234
|
+
/** The `base_freeze` table exactly as SQLite hands it back. */
|
|
235
|
+
interface BaseFreezeRow {
|
|
236
|
+
project: string;
|
|
237
|
+
repo: string;
|
|
238
|
+
culpritSha: string;
|
|
239
|
+
detail: string | null;
|
|
240
|
+
setAt: number;
|
|
241
|
+
clearedAt: number | null;
|
|
242
|
+
clearedBy: string | null;
|
|
243
|
+
clearedReason: string | null;
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
function toBaseFreeze(row: BaseFreezeRow): BaseFreeze {
|
|
247
|
+
const freeze: BaseFreeze = {
|
|
248
|
+
repo: row.repo,
|
|
249
|
+
culpritSha: row.culpritSha,
|
|
250
|
+
setAt: row.setAt,
|
|
251
|
+
};
|
|
252
|
+
if (row.detail !== null) freeze.detail = row.detail;
|
|
253
|
+
if (row.clearedAt !== null) {
|
|
254
|
+
freeze.clearedAt = row.clearedAt;
|
|
255
|
+
if (row.clearedBy !== null) freeze.clearedBy = row.clearedBy;
|
|
256
|
+
if (row.clearedReason !== null) freeze.clearedReason = row.clearedReason;
|
|
257
|
+
}
|
|
258
|
+
return freeze;
|
|
259
|
+
}
|
|
260
|
+
|
|
207
261
|
interface FrictionRollupRow {
|
|
208
262
|
project: string;
|
|
209
263
|
day: string;
|
|
@@ -264,7 +318,10 @@ interface ReportRow {
|
|
|
264
318
|
updatedAt: number;
|
|
265
319
|
nextAttemptAt: number;
|
|
266
320
|
messageId: number | null;
|
|
321
|
+
messageIds: string | null;
|
|
267
322
|
deliveredAt: number | null;
|
|
323
|
+
sentParts: number;
|
|
324
|
+
sentPartsHash: string | null;
|
|
268
325
|
lastError: string | null;
|
|
269
326
|
}
|
|
270
327
|
|
|
@@ -404,6 +461,14 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
404
461
|
turns INTEGER NOT NULL,
|
|
405
462
|
maxTurns INTEGER NOT NULL,
|
|
406
463
|
spendUsd REAL NOT NULL,
|
|
464
|
+
provider429Count INTEGER,
|
|
465
|
+
resolvedModel TEXT,
|
|
466
|
+
resolvedProvider TEXT,
|
|
467
|
+
retryFallbacks TEXT,
|
|
468
|
+
retryFallbackSucceeded INTEGER,
|
|
469
|
+
modelRecoveries INTEGER,
|
|
470
|
+
autoRetryCount INTEGER,
|
|
471
|
+
autoCompactionCount INTEGER,
|
|
407
472
|
sessionFile TEXT,
|
|
408
473
|
prUrl TEXT,
|
|
409
474
|
headSha TEXT,
|
|
@@ -439,6 +504,23 @@ CREATE TABLE IF NOT EXISTS base_health (
|
|
|
439
504
|
PRIMARY KEY (project, repo)
|
|
440
505
|
);
|
|
441
506
|
|
|
507
|
+
-- A per-repo merge freeze (#283). While clearedAt IS NULL the repo is frozen:
|
|
508
|
+
-- prMergeVerb refuses further merges to it with the base-red-freeze refusal. Set
|
|
509
|
+
-- when a watched merge (or live base observation) turns the base red; the clear
|
|
510
|
+
-- holds the mechanical green recovery and the operator's unfreeze verb, and acts
|
|
511
|
+
-- as the durable override/clear audit ledger.
|
|
512
|
+
CREATE TABLE IF NOT EXISTS base_freeze (
|
|
513
|
+
project TEXT NOT NULL,
|
|
514
|
+
repo TEXT NOT NULL,
|
|
515
|
+
culpritSha TEXT NOT NULL,
|
|
516
|
+
detail TEXT,
|
|
517
|
+
setAt INTEGER NOT NULL,
|
|
518
|
+
clearedAt INTEGER,
|
|
519
|
+
clearedBy TEXT,
|
|
520
|
+
clearedReason TEXT,
|
|
521
|
+
PRIMARY KEY (project, repo)
|
|
522
|
+
);
|
|
523
|
+
|
|
442
524
|
CREATE TABLE IF NOT EXISTS notifications (
|
|
443
525
|
"key" TEXT PRIMARY KEY,
|
|
444
526
|
at INTEGER NOT NULL
|
|
@@ -516,7 +598,10 @@ CREATE TABLE IF NOT EXISTS reports (
|
|
|
516
598
|
updatedAt INTEGER NOT NULL,
|
|
517
599
|
nextAttemptAt INTEGER NOT NULL,
|
|
518
600
|
messageId INTEGER,
|
|
601
|
+
messageIds TEXT,
|
|
519
602
|
deliveredAt INTEGER,
|
|
603
|
+
sentParts INTEGER NOT NULL DEFAULT 0,
|
|
604
|
+
sentPartsHash TEXT,
|
|
520
605
|
lastError TEXT
|
|
521
606
|
);
|
|
522
607
|
CREATE UNIQUE INDEX IF NOT EXISTS reports_dedupe
|
|
@@ -624,6 +709,7 @@ CREATE TABLE IF NOT EXISTS merge_locks (
|
|
|
624
709
|
CREATE TABLE IF NOT EXISTS decisions (
|
|
625
710
|
id TEXT PRIMARY KEY,
|
|
626
711
|
project TEXT NOT NULL,
|
|
712
|
+
kind TEXT NOT NULL DEFAULT 'question',
|
|
627
713
|
question TEXT NOT NULL,
|
|
628
714
|
blocks TEXT,
|
|
629
715
|
askedAt INTEGER NOT NULL,
|
|
@@ -756,6 +842,29 @@ function toSql(value: unknown): SqlValue {
|
|
|
756
842
|
* row hand-edited or written by a future version drops its flags rather than
|
|
757
843
|
* throwing and taking `status` down with it.
|
|
758
844
|
*/
|
|
845
|
+
/**
|
|
846
|
+
* Within-run fallback pairs out of their JSON column, or undefined for
|
|
847
|
+
* anything that is not an array of from/to string pairs. Same defensive
|
|
848
|
+
* posture as {@link toSettlementFlags}: a row hand-edited or written by a
|
|
849
|
+
* future version drops its fallbacks rather than throwing.
|
|
850
|
+
*/
|
|
851
|
+
function toRetryFallbacks(text: string): { from: string; to: string }[] | undefined {
|
|
852
|
+
try {
|
|
853
|
+
const value: unknown = JSON.parse(text);
|
|
854
|
+
if (!Array.isArray(value) || value.length === 0) return undefined;
|
|
855
|
+
const pairs = value.filter(
|
|
856
|
+
(pair): pair is { from: string; to: string } =>
|
|
857
|
+
typeof pair === "object" &&
|
|
858
|
+
pair !== null &&
|
|
859
|
+
typeof (pair as { from?: unknown }).from === "string" &&
|
|
860
|
+
typeof (pair as { to?: unknown }).to === "string",
|
|
861
|
+
);
|
|
862
|
+
return pairs.length === 0 ? undefined : pairs;
|
|
863
|
+
} catch {
|
|
864
|
+
return undefined;
|
|
865
|
+
}
|
|
866
|
+
}
|
|
867
|
+
|
|
759
868
|
function toSettlementFlags(text: string): SettlementFlag[] | undefined {
|
|
760
869
|
try {
|
|
761
870
|
const value: unknown = JSON.parse(text);
|
|
@@ -810,6 +919,17 @@ function toRecord(row: RunRow): RunRecord {
|
|
|
810
919
|
if (flags !== undefined) record.settlementFlags = flags;
|
|
811
920
|
}
|
|
812
921
|
if (row.report !== null) record.report = row.report;
|
|
922
|
+
if (row.provider429Count !== null) record.provider429Count = row.provider429Count;
|
|
923
|
+
if (row.resolvedModel !== null) record.resolvedModel = row.resolvedModel;
|
|
924
|
+
if (row.resolvedProvider !== null) record.resolvedProvider = row.resolvedProvider;
|
|
925
|
+
if (row.retryFallbacks !== null) {
|
|
926
|
+
const fallbacks = toRetryFallbacks(row.retryFallbacks);
|
|
927
|
+
if (fallbacks !== undefined) record.retryFallbacks = fallbacks;
|
|
928
|
+
}
|
|
929
|
+
if (row.retryFallbackSucceeded !== null) record.retryFallbackSucceeded = row.retryFallbackSucceeded;
|
|
930
|
+
if (row.modelRecoveries !== null) record.modelRecoveries = row.modelRecoveries;
|
|
931
|
+
if (row.autoRetryCount !== null) record.autoRetryCount = row.autoRetryCount;
|
|
932
|
+
if (row.autoCompactionCount !== null) record.autoCompactionCount = row.autoCompactionCount;
|
|
813
933
|
if (row.failureClass !== null) record.failureClass = row.failureClass as FailureClass;
|
|
814
934
|
if (row.recoveryAction !== null) record.recoveryAction = row.recoveryAction as RecoveryAction;
|
|
815
935
|
if (row.recoveredAt !== null) record.recoveredAt = row.recoveredAt;
|
|
@@ -837,6 +957,18 @@ function toReport(row: ReportRow): ReportRecord {
|
|
|
837
957
|
if (row.attemptId !== null) record.attemptId = row.attemptId;
|
|
838
958
|
if (row.dedupeKey !== null) record.dedupeKey = row.dedupeKey;
|
|
839
959
|
if (row.messageId !== null) record.messageId = row.messageId;
|
|
960
|
+
if (row.messageIds !== null) {
|
|
961
|
+
// Written as JSON by this store; a row that fails to parse reads as if
|
|
962
|
+
// no ids were recorded, which is the honest reading of unreadable bytes.
|
|
963
|
+
try {
|
|
964
|
+
const ids: unknown = JSON.parse(row.messageIds);
|
|
965
|
+
if (Array.isArray(ids) && ids.every((id) => typeof id === "number")) record.messageIds = ids;
|
|
966
|
+
} catch {
|
|
967
|
+
// treated as absent
|
|
968
|
+
}
|
|
969
|
+
}
|
|
970
|
+
if (row.sentParts !== 0) record.sentParts = row.sentParts;
|
|
971
|
+
if (row.sentPartsHash !== null) record.sentPartsHash = row.sentPartsHash;
|
|
840
972
|
if (row.deliveredAt !== null) record.deliveredAt = row.deliveredAt;
|
|
841
973
|
if (row.lastError !== null) record.lastError = row.lastError;
|
|
842
974
|
return record;
|
|
@@ -858,6 +990,7 @@ function toMaterialEvent(row: MaterialEventRow): MaterialEvent {
|
|
|
858
990
|
interface DecisionRow {
|
|
859
991
|
id: string;
|
|
860
992
|
project: string;
|
|
993
|
+
kind: string;
|
|
861
994
|
question: string;
|
|
862
995
|
blocks: string | null;
|
|
863
996
|
askedAt: number;
|
|
@@ -874,6 +1007,10 @@ function toDecision(row: DecisionRow): DecisionRecord {
|
|
|
874
1007
|
const record: DecisionRecord = {
|
|
875
1008
|
id: row.id,
|
|
876
1009
|
project: row.project,
|
|
1010
|
+
// Only `watch` is ever a watch: anything else — and especially the historic
|
|
1011
|
+
// NULL the migration backfilled as `question` — reads as a human question.
|
|
1012
|
+
// A real question may carry a condition; that must not relabel it.
|
|
1013
|
+
kind: row.kind === "watch" ? "watch" : "question",
|
|
877
1014
|
question: row.question,
|
|
878
1015
|
askedAt: row.askedAt,
|
|
879
1016
|
expiresAt: row.expiresAt,
|
|
@@ -949,7 +1086,11 @@ function toDispatchSummary(text: string): DispatchSummary | undefined {
|
|
|
949
1086
|
!Array.isArray(hold.issues) ||
|
|
950
1087
|
hold.issues.length > 5 ||
|
|
951
1088
|
hold.count < hold.issues.length ||
|
|
952
|
-
!hold.issues.every((issue) => Number.isSafeInteger(issue) && issue > 0)
|
|
1089
|
+
!hold.issues.every((issue) => Number.isSafeInteger(issue) && issue > 0) ||
|
|
1090
|
+
(hold.details !== undefined &&
|
|
1091
|
+
(!Array.isArray(hold.details) ||
|
|
1092
|
+
hold.details.length > 5 ||
|
|
1093
|
+
!hold.details.every((detail) => typeof detail === "string" && detail.length > 0))),
|
|
953
1094
|
)
|
|
954
1095
|
) {
|
|
955
1096
|
return undefined;
|
|
@@ -1002,6 +1143,51 @@ export function dbPath(): string {
|
|
|
1002
1143
|
return join(stateDir(), "conductor.db");
|
|
1003
1144
|
}
|
|
1004
1145
|
|
|
1146
|
+
/** The stem every restorable `conductor.db` snapshot is published under. */
|
|
1147
|
+
export const DB_SNAPSHOT_STEM = "conductor.db.bak";
|
|
1148
|
+
|
|
1149
|
+
/**
|
|
1150
|
+
* A fresh, restorable copy of the store at `source`, published under `destDir`
|
|
1151
|
+
* with the same temp-then-atomic-publish convention as the config backups.
|
|
1152
|
+
*
|
|
1153
|
+
* Uses SQLite's own `VACUUM INTO`, never a plain file copy: the store opens
|
|
1154
|
+
* with `journal_mode=WAL`, and a `copyFileSync` can capture a torn database
|
|
1155
|
+
* while committed rows still sit uncheckpointed in the live WAL — a file that
|
|
1156
|
+
* exists, looks fresh and passes `integrity_check` yet is missing the newest
|
|
1157
|
+
* commits. `VACUUM INTO` reads the source as a whole (main file plus WAL) and
|
|
1158
|
+
* writes a single-file snapshot, so the result opens cleanly in a fresh
|
|
1159
|
+
* connection and contains every committed row.
|
|
1160
|
+
*
|
|
1161
|
+
* The snapshot runs against its own read of the source, deliberately not the
|
|
1162
|
+
* caller's writer connection, so a snapshot can be taken while a writer holds
|
|
1163
|
+
* an uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
|
|
1164
|
+
* refused.
|
|
1165
|
+
*
|
|
1166
|
+
* Returns the published snapshot path.
|
|
1167
|
+
*/
|
|
1168
|
+
export function snapshotDb(source: string, destDir: string): string {
|
|
1169
|
+
if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
|
|
1170
|
+
if (!existsSync(source)) {
|
|
1171
|
+
throw new Error(`cannot snapshot ${source}: the store does not exist`);
|
|
1172
|
+
}
|
|
1173
|
+
mkdirSync(destDir, { recursive: true });
|
|
1174
|
+
const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
|
|
1175
|
+
let connection: Database | undefined;
|
|
1176
|
+
try {
|
|
1177
|
+
connection = new Database(source);
|
|
1178
|
+
connection.exec("PRAGMA busy_timeout = 5000;");
|
|
1179
|
+
// `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
|
|
1180
|
+
connection.exec(`VACUUM INTO '${temporary.replaceAll("'", "''")}'`);
|
|
1181
|
+
} catch (err) {
|
|
1182
|
+
// Never leave a half-written snapshot on disk.
|
|
1183
|
+
rmSync(temporary, { force: true });
|
|
1184
|
+
throw err;
|
|
1185
|
+
} finally {
|
|
1186
|
+
connection?.close();
|
|
1187
|
+
}
|
|
1188
|
+
return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp()}`);
|
|
1189
|
+
}
|
|
1190
|
+
|
|
1005
1191
|
/** The UTC calendar day (`YYYY-MM-DD`) a moment falls on — the partition key
|
|
1006
1192
|
* for the daemon's observed github call counter (#198). */
|
|
1007
1193
|
export function utcDay(now: number = Date.now()): string {
|
|
@@ -1120,6 +1306,30 @@ export function openStore(dbPath: string): Store {
|
|
|
1120
1306
|
if (!columns.some((column) => column.name === "model")) {
|
|
1121
1307
|
db.exec("ALTER TABLE runs ADD COLUMN model TEXT");
|
|
1122
1308
|
}
|
|
1309
|
+
// The count of in-session provider 429s a run recorded was first written by
|
|
1310
|
+
// the worker (#573). Rows predating the column are NULL, which is the honest
|
|
1311
|
+
// reading: the count was not recorded, so the classifier must not treat the
|
|
1312
|
+
// absence as "no rate limiting happened".
|
|
1313
|
+
if (!columns.some((column) => column.name === "provider429Count")) {
|
|
1314
|
+
db.exec("ALTER TABLE runs ADD COLUMN provider429Count INTEGER");
|
|
1315
|
+
}
|
|
1316
|
+
// The within-run harness reliability surface (#539/#581): the resolved
|
|
1317
|
+
// model/provider and the fallback/retry/compaction counters. Rows predating
|
|
1318
|
+
// the columns are NULL, the honest reading of "the collection side had not
|
|
1319
|
+
// landed yet" — never "the run was healthy".
|
|
1320
|
+
for (const [name, type] of [
|
|
1321
|
+
["resolvedModel", "TEXT"],
|
|
1322
|
+
["resolvedProvider", "TEXT"],
|
|
1323
|
+
["retryFallbacks", "TEXT"],
|
|
1324
|
+
["retryFallbackSucceeded", "INTEGER"],
|
|
1325
|
+
["modelRecoveries", "INTEGER"],
|
|
1326
|
+
["autoRetryCount", "INTEGER"],
|
|
1327
|
+
["autoCompactionCount", "INTEGER"],
|
|
1328
|
+
] as const) {
|
|
1329
|
+
if (!columns.some((column) => column.name === name)) {
|
|
1330
|
+
db.exec(`ALTER TABLE runs ADD COLUMN ${name} ${type}`);
|
|
1331
|
+
}
|
|
1332
|
+
}
|
|
1123
1333
|
// Existing held-notice rows predate exact digest ownership (#274). A NULL
|
|
1124
1334
|
// report id is truthful for already-digested history and means "still owed"
|
|
1125
1335
|
// only while digestedAt is also NULL.
|
|
@@ -1138,6 +1348,30 @@ export function openStore(dbPath: string): Store {
|
|
|
1138
1348
|
if (!heldNoticeColumns.some((column) => column.name === "urgent")) {
|
|
1139
1349
|
db.exec("ALTER TABLE held_notices ADD COLUMN urgent INTEGER NOT NULL DEFAULT 0");
|
|
1140
1350
|
}
|
|
1351
|
+
// Multi-part delivery (#566): reports created before splitting existed carry
|
|
1352
|
+
// a single `messageId` and no per-part state, which is the honest reading of
|
|
1353
|
+
// a row that predates the watermark. `sentParts` defaults to 0 — "nothing
|
|
1354
|
+
// confirmed" — so a retry of a historical row starts from the first part.
|
|
1355
|
+
const reportColumns = db.query<{ name: string }, []>("PRAGMA table_info(reports)").all();
|
|
1356
|
+
if (!reportColumns.some((column) => column.name === "messageIds")) {
|
|
1357
|
+
db.exec("ALTER TABLE reports ADD COLUMN messageIds TEXT");
|
|
1358
|
+
}
|
|
1359
|
+
if (!reportColumns.some((column) => column.name === "sentParts")) {
|
|
1360
|
+
db.exec("ALTER TABLE reports ADD COLUMN sentParts INTEGER NOT NULL DEFAULT 0");
|
|
1361
|
+
}
|
|
1362
|
+
if (!reportColumns.some((column) => column.name === "sentPartsHash")) {
|
|
1363
|
+
db.exec("ALTER TABLE reports ADD COLUMN sentPartsHash TEXT");
|
|
1364
|
+
}
|
|
1365
|
+
// The kind split (#459): a row is either a question a human must answer or a
|
|
1366
|
+
// watch the orchestrator set for itself. Databases written before the split
|
|
1367
|
+
// never marked it, and every such row was a question — `watch` has no
|
|
1368
|
+
// historical form, because the old `decision open --resolves-when` was the
|
|
1369
|
+
// orchestrator's own misjudgement, not a durable kind. Defaulting the
|
|
1370
|
+
// historical rows to `question` is the honest reading.
|
|
1371
|
+
const decisionColumns = db.query<{ name: string }, []>("PRAGMA table_info(decisions)").all();
|
|
1372
|
+
if (!decisionColumns.some((column) => column.name === "kind")) {
|
|
1373
|
+
db.exec("ALTER TABLE decisions ADD COLUMN kind TEXT NOT NULL DEFAULT 'question'");
|
|
1374
|
+
}
|
|
1141
1375
|
|
|
1142
1376
|
// One-time repair, and the reason it lives here rather than in the classifier:
|
|
1143
1377
|
// a turn-0 environment fault that was ALREADY classified `unknown` and
|
|
@@ -1323,6 +1557,43 @@ export function openStore(dbPath: string): Store {
|
|
|
1323
1557
|
GROUP BY repo, baseRef
|
|
1324
1558
|
ORDER BY repo ASC, baseRef ASC`,
|
|
1325
1559
|
);
|
|
1560
|
+
// Re-open (or refresh) an active freeze. On conflict we reset the clear
|
|
1561
|
+
// columns — a red re-observation after an override must re-arm the freeze —
|
|
1562
|
+
// and take the newest culprit/evidence/setAt as the current truth.
|
|
1563
|
+
const setBaseFreezeRow = db.query<
|
|
1564
|
+
unknown,
|
|
1565
|
+
[string, string, string, SqlValue, number]
|
|
1566
|
+
>(
|
|
1567
|
+
`INSERT INTO base_freeze
|
|
1568
|
+
(project, repo, culpritSha, detail, setAt, clearedAt, clearedBy, clearedReason)
|
|
1569
|
+
VALUES (?, ?, ?, ?, ?, NULL, NULL, NULL)
|
|
1570
|
+
ON CONFLICT(project, repo) DO UPDATE SET
|
|
1571
|
+
culpritSha = excluded.culpritSha,
|
|
1572
|
+
detail = excluded.detail,
|
|
1573
|
+
setAt = excluded.setAt,
|
|
1574
|
+
clearedAt = NULL,
|
|
1575
|
+
clearedBy = NULL,
|
|
1576
|
+
clearedReason = NULL`,
|
|
1577
|
+
);
|
|
1578
|
+
const wasBaseFrozen = db.query<{ n: number }, [string, string]>(
|
|
1579
|
+
`SELECT COUNT(*) AS n FROM base_freeze
|
|
1580
|
+
WHERE project = ? AND repo = ? AND clearedAt IS NULL`,
|
|
1581
|
+
);
|
|
1582
|
+
const selectBaseFreeze = db.query<BaseFreezeRow, [string, string]>(
|
|
1583
|
+
`SELECT * FROM base_freeze WHERE project = ? AND repo = ?`,
|
|
1584
|
+
);
|
|
1585
|
+
const selectFreezes = db.query<BaseFreezeRow, [string]>(
|
|
1586
|
+
`SELECT * FROM base_freeze WHERE project = ?
|
|
1587
|
+
ORDER BY (clearedAt IS NULL) DESC, setAt DESC, repo ASC`,
|
|
1588
|
+
);
|
|
1589
|
+
const clearBaseFreezeRow = db.query<
|
|
1590
|
+
unknown,
|
|
1591
|
+
[number, string, string, string, string]
|
|
1592
|
+
>(
|
|
1593
|
+
`UPDATE base_freeze
|
|
1594
|
+
SET clearedAt = ?, clearedBy = ?, clearedReason = ?
|
|
1595
|
+
WHERE project = ? AND repo = ? AND clearedAt IS NULL`,
|
|
1596
|
+
);
|
|
1326
1597
|
const countAttempts = db.query<{ n: number }, [string, number]>(
|
|
1327
1598
|
`SELECT COUNT(*) AS n FROM runs WHERE project = ? AND issue = ?`,
|
|
1328
1599
|
);
|
|
@@ -1335,14 +1606,14 @@ export function openStore(dbPath: string): Store {
|
|
|
1335
1606
|
const countFailures = db.query<{ n: number }, [string, number]>(
|
|
1336
1607
|
`SELECT COUNT(*) AS n FROM runs
|
|
1337
1608
|
WHERE project = ? AND issue = ? AND state = 'failed'
|
|
1338
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'returned-for-revision'))`,
|
|
1609
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('ci-infra', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity', 'returned-for-revision'))`,
|
|
1339
1610
|
);
|
|
1340
1611
|
const countContinuations = db.query<{ n: number }, [string, number]>(
|
|
1341
1612
|
`SELECT COUNT(*) AS n FROM runs
|
|
1342
1613
|
WHERE project = ? AND issue = ?
|
|
1343
1614
|
AND (
|
|
1344
1615
|
(state IN ('killed', 'orphaned', 'blocked')
|
|
1345
|
-
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient')))
|
|
1616
|
+
AND (failureClass IS NULL OR failureClass NOT IN ('admin-kill', 'settlement-stuck', 'env-start-failure', 'dispatch-infra', 'provider-credit', 'provider-transient', 'provider-capacity')))
|
|
1346
1617
|
OR (state = 'failed' AND failureClass = 'returned-for-revision')
|
|
1347
1618
|
)`,
|
|
1348
1619
|
);
|
|
@@ -1840,10 +2111,19 @@ export function openStore(dbPath: string): Store {
|
|
|
1840
2111
|
);
|
|
1841
2112
|
const deliverReportRow = db.query<unknown, SqlValue[]>(
|
|
1842
2113
|
`UPDATE reports
|
|
1843
|
-
SET state = 'delivered', attemptId = NULL, messageId = ?, deliveredAt = ?,
|
|
2114
|
+
SET state = 'delivered', attemptId = NULL, messageId = ?, messageIds = ?, deliveredAt = ?,
|
|
1844
2115
|
updatedAt = ?, lastError = NULL
|
|
1845
2116
|
WHERE id = ? AND state = 'sending' AND attemptId = ?`,
|
|
1846
2117
|
);
|
|
2118
|
+
// Mid-split progress, written before the next part ships: the first
|
|
2119
|
+
// `sentParts` parts of the fingerprinted message are confirmed accepted, so
|
|
2120
|
+
// a crash or a definitive failure can only ever re-send the part that was in
|
|
2121
|
+
// flight (#566). Guarded by attempt id like every other transition.
|
|
2122
|
+
const markPartsSentRow = db.query<unknown, [number, string, number, string, string]>(
|
|
2123
|
+
`UPDATE reports
|
|
2124
|
+
SET sentParts = ?, sentPartsHash = ?, updatedAt = ?
|
|
2125
|
+
WHERE id = ? AND state = 'sending' AND attemptId = ?`,
|
|
2126
|
+
);
|
|
1847
2127
|
const requeueReportRow = db.query<unknown, [number, string, number, string, string]>(
|
|
1848
2128
|
`UPDATE reports
|
|
1849
2129
|
SET state = 'pending', attemptId = NULL, nextAttemptAt = ?, lastError = ?, updatedAt = ?
|
|
@@ -1962,10 +2242,10 @@ export function openStore(dbPath: string): Store {
|
|
|
1962
2242
|
},
|
|
1963
2243
|
);
|
|
1964
2244
|
|
|
1965
|
-
const insertDecision = db.query<never, [string, string, string, SqlValue, number, number, SqlValue, string]>(
|
|
2245
|
+
const insertDecision = db.query<never, [string, string, string, string, SqlValue, number, number, SqlValue, string]>(
|
|
1966
2246
|
`INSERT INTO decisions
|
|
1967
|
-
(id, project, question, blocks, askedAt, expiresAt, condition, state)
|
|
1968
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
2247
|
+
(id, project, kind, question, blocks, askedAt, expiresAt, condition, state)
|
|
2248
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
1969
2249
|
);
|
|
1970
2250
|
const selectDecision = db.query<DecisionRow, [string]>(`SELECT * FROM decisions WHERE id = ?`);
|
|
1971
2251
|
const selectOpenDecisions = db.query<DecisionRow, [string]>(
|
|
@@ -2276,6 +2556,41 @@ export function openStore(dbPath: string): Store {
|
|
|
2276
2556
|
);
|
|
2277
2557
|
},
|
|
2278
2558
|
|
|
2559
|
+
baseFreeze(project: string, repo: string): BaseFreeze | undefined {
|
|
2560
|
+
const row = selectBaseFreeze.get(project, repo);
|
|
2561
|
+
return row === null ? undefined : toBaseFreeze(row);
|
|
2562
|
+
},
|
|
2563
|
+
|
|
2564
|
+
freezes(project: string): BaseFreeze[] {
|
|
2565
|
+
return selectFreezes.all(project).map(toBaseFreeze);
|
|
2566
|
+
},
|
|
2567
|
+
|
|
2568
|
+
setBaseFreeze(
|
|
2569
|
+
project: string,
|
|
2570
|
+
freeze: { repo: string; culpritSha: string; detail?: string; setAt?: number },
|
|
2571
|
+
): boolean {
|
|
2572
|
+
const setAt = freeze.setAt ?? Date.now();
|
|
2573
|
+
const wasActive = (wasBaseFrozen.get(project, freeze.repo)?.n ?? 0) > 0;
|
|
2574
|
+
setBaseFreezeRow.run(
|
|
2575
|
+
project,
|
|
2576
|
+
freeze.repo,
|
|
2577
|
+
freeze.culpritSha,
|
|
2578
|
+
freeze.detail ?? null,
|
|
2579
|
+
setAt,
|
|
2580
|
+
);
|
|
2581
|
+
return !wasActive;
|
|
2582
|
+
},
|
|
2583
|
+
|
|
2584
|
+
clearBaseFreeze(
|
|
2585
|
+
project: string,
|
|
2586
|
+
repo: string,
|
|
2587
|
+
by: string,
|
|
2588
|
+
reason: string,
|
|
2589
|
+
at = Date.now(),
|
|
2590
|
+
): boolean {
|
|
2591
|
+
return clearBaseFreezeRow.run(at, by, reason, project, repo).changes > 0;
|
|
2592
|
+
},
|
|
2593
|
+
|
|
2279
2594
|
|
|
2280
2595
|
salvagedRuns(project: string): RunRecord[] {
|
|
2281
2596
|
return selectSalvaged.all(project).map(toRecord);
|
|
@@ -2660,14 +2975,33 @@ export function openStore(dbPath: string): Store {
|
|
|
2660
2975
|
markReportDelivered(
|
|
2661
2976
|
id: string,
|
|
2662
2977
|
attemptId: string,
|
|
2663
|
-
|
|
2978
|
+
messageIds: readonly number[],
|
|
2664
2979
|
at: number,
|
|
2665
2980
|
): boolean {
|
|
2981
|
+
// `messageId` stays the first accepted id so rows written before
|
|
2982
|
+
// multi-part sends and the operator-facing rendering never disagree.
|
|
2666
2983
|
return (
|
|
2667
|
-
deliverReportRow.run(
|
|
2984
|
+
deliverReportRow.run(
|
|
2985
|
+
toSql(messageIds[0]),
|
|
2986
|
+
toSql(messageIds.length === 0 ? null : JSON.stringify(messageIds)),
|
|
2987
|
+
at,
|
|
2988
|
+
at,
|
|
2989
|
+
id,
|
|
2990
|
+
attemptId,
|
|
2991
|
+
).changes > 0
|
|
2668
2992
|
);
|
|
2669
2993
|
},
|
|
2670
2994
|
|
|
2995
|
+
markReportPartsSent(
|
|
2996
|
+
id: string,
|
|
2997
|
+
attemptId: string,
|
|
2998
|
+
sentParts: number,
|
|
2999
|
+
sentPartsHash: string,
|
|
3000
|
+
at: number,
|
|
3001
|
+
): boolean {
|
|
3002
|
+
return markPartsSentRow.run(sentParts, sentPartsHash, at, id, attemptId).changes > 0;
|
|
3003
|
+
},
|
|
3004
|
+
|
|
2671
3005
|
markReportUncertain(id: string, attemptId: string, error: string): boolean {
|
|
2672
3006
|
return uncertainReportRow.run(error, id, attemptId).changes > 0;
|
|
2673
3007
|
},
|
|
@@ -2743,12 +3077,18 @@ export function openStore(dbPath: string): Store {
|
|
|
2743
3077
|
},
|
|
2744
3078
|
|
|
2745
3079
|
createDecision(draft: DecisionDraft): DecisionRecord {
|
|
3080
|
+
const kind = draft.kind ?? "question";
|
|
2746
3081
|
const record: DecisionRecord = {
|
|
2747
3082
|
id: crypto.randomUUID(),
|
|
2748
3083
|
project: draft.project,
|
|
3084
|
+
kind,
|
|
2749
3085
|
question: draft.question,
|
|
2750
3086
|
askedAt: draft.at,
|
|
2751
|
-
|
|
3087
|
+
// The seven-day deadline is for a question a human never answered. A
|
|
3088
|
+
// watch has no such deadline: it is the orchestrator's own bookkeeping,
|
|
3089
|
+
// and expiring it silently could drop a release the day its condition
|
|
3090
|
+
// finally fires, so it is never due.
|
|
3091
|
+
expiresAt: kind === "watch" ? DECISION_WATCH_NEVER_EXPIRES : draft.at + DECISION_TTL_MS,
|
|
2752
3092
|
state: "open",
|
|
2753
3093
|
};
|
|
2754
3094
|
if (draft.blocks !== undefined) record.blocks = draft.blocks;
|
|
@@ -2756,6 +3096,7 @@ export function openStore(dbPath: string): Store {
|
|
|
2756
3096
|
insertDecision.run(
|
|
2757
3097
|
record.id,
|
|
2758
3098
|
record.project,
|
|
3099
|
+
record.kind,
|
|
2759
3100
|
record.question,
|
|
2760
3101
|
toSql(record.blocks),
|
|
2761
3102
|
record.askedAt,
|