omp-conductor 0.16.2 → 0.17.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +38 -4
- package/REFERENCE.md +18 -12
- package/package.json +2 -1
- package/schema/config.schema.json +16 -0
- package/src/admission.ts +159 -43
- package/src/availability.ts +27 -1
- package/src/briefs/worker.md +2 -0
- package/src/clack-ui.ts +83 -0
- package/src/command-manifest.ts +16 -7
- package/src/commands/arm.ts +11 -3
- package/src/commands/decision.ts +17 -7
- package/src/commands/doctor.ts +18 -1
- package/src/commands/hold.ts +9 -7
- package/src/commands/ledger.ts +25 -4
- package/src/commands/message.ts +32 -4
- package/src/commands/setup.ts +61 -10
- package/src/commands/stats.ts +9 -5
- package/src/commands/status.ts +32 -5
- package/src/commands/tail.ts +13 -1
- package/src/commands/watch.ts +16 -7
- package/src/config-schema.ts +20 -0
- package/src/config.ts +37 -0
- package/src/daemon.ts +1240 -18
- package/src/doctor.ts +310 -22
- package/src/escalate.ts +560 -57
- package/src/failure-class.ts +56 -13
- package/src/fleet.ts +224 -47
- package/src/gitops.ts +103 -24
- package/src/lifecycle.ts +7 -2
- package/src/orchestrator-tick.ts +372 -157
- package/src/privileged.ts +3 -0
- package/src/release-policy.ts +177 -5
- package/src/setup-answers.ts +135 -0
- package/src/setup-host.ts +193 -4
- package/src/setup-install.ts +2 -0
- package/src/setup-probe.ts +1 -0
- package/src/setup-wizard.ts +1296 -101
- package/src/setup.ts +60 -3
- package/src/status-render.ts +11 -1
- package/src/store.ts +333 -12
- package/src/tracker/github.ts +562 -13
- package/src/types.ts +204 -2
- package/src/ui/progress.ts +32 -0
- package/src/ui/style.ts +11 -0
- package/src/upgrade.ts +50 -19
- package/src/verbs/actions.ts +66 -18
- package/src/verbs/protocol.ts +45 -0
- package/src/verbs/server.ts +212 -11
- package/src/wizard-ui.ts +14 -5
- package/src/worker.ts +26 -0
- package/systemd/omp-conductor-recover.sh +73 -0
- package/systemd/recover-unit-test.sh +61 -0
package/src/setup.ts
CHANGED
|
@@ -8,7 +8,8 @@
|
|
|
8
8
|
* least tested code in the package.
|
|
9
9
|
*
|
|
10
10
|
* Two functions here mutate something outside the process: `createMissingLabels`
|
|
11
|
-
* and
|
|
11
|
+
* (and its rollback compensation half, `deleteCreatedLabels`) and
|
|
12
|
+
* `writeOrchestratorBrief`. Everything else reads, or computes. That is what
|
|
12
13
|
* lets the plugin show a complete plan before asking for consent, and it is a
|
|
13
14
|
* property worth preserving — check it before adding a function.
|
|
14
15
|
*
|
|
@@ -40,6 +41,7 @@ import {
|
|
|
40
41
|
configPath,
|
|
41
42
|
defaultMirrorRoot,
|
|
42
43
|
defaultWorkspaceRoot,
|
|
44
|
+
resolveArmProof,
|
|
43
45
|
resolveCaps,
|
|
44
46
|
resolvePolicy,
|
|
45
47
|
resolveReleaseGrants,
|
|
@@ -50,6 +52,7 @@ import { graphProjectPath, graphRepos } from "./graph.ts";
|
|
|
50
52
|
import { ompSettingsOverlay } from "./omp-settings.ts";
|
|
51
53
|
import {
|
|
52
54
|
CONFIG_VERSION,
|
|
55
|
+
DEFAULT_ARM_PROOF,
|
|
53
56
|
DEFAULT_AUTHORITY,
|
|
54
57
|
DEFAULT_CAPS,
|
|
55
58
|
DEFAULT_PROJECT_POLICY,
|
|
@@ -57,6 +60,7 @@ import {
|
|
|
57
60
|
DENIED_RELEASE_GRANTS,
|
|
58
61
|
RELEASE_SHAPES,
|
|
59
62
|
WEEKDAYS,
|
|
63
|
+
type ArmProof,
|
|
60
64
|
type BaseFreshness,
|
|
61
65
|
type BehindBaseAction,
|
|
62
66
|
type AuthorityHolder,
|
|
@@ -205,6 +209,12 @@ export interface SetupAnswers {
|
|
|
205
209
|
* leave one to be defaulted by whichever reader gets there first.
|
|
206
210
|
*/
|
|
207
211
|
policy: ProjectPolicy;
|
|
212
|
+
/**
|
|
213
|
+
* How `arm` proves a human just approved arming (#613). Always complete:
|
|
214
|
+
* the wizard asks about it in the policy area, so an answers object can never
|
|
215
|
+
* leave it to be defaulted by whichever reader gets there first.
|
|
216
|
+
*/
|
|
217
|
+
armProof: ArmProof;
|
|
208
218
|
/**
|
|
209
219
|
* Hand-edited recovery merge authorizations carried through setup unchanged.
|
|
210
220
|
* The wizard never grants one; forgetting them during an unrelated amend
|
|
@@ -381,8 +391,23 @@ export const BEHIND_BASE_CHOICES: { readonly [K in BehindBaseAction]: string } =
|
|
|
381
391
|
escalate: "page a human rather than guess",
|
|
382
392
|
};
|
|
383
393
|
|
|
394
|
+
/**
|
|
395
|
+
* What each arming proof means to the operator being asked about it, in their
|
|
396
|
+
* words rather than the gate's (conductor #613). The `claim-only` consequence
|
|
397
|
+
* is the one line the issue contract requires the question to state: anything
|
|
398
|
+
* that can invoke the already-privileged `omp-conductor arm` command can start
|
|
399
|
+
* dispatch once the live claim and poller pass. Mapped over the closed union
|
|
400
|
+
* so a third proof fails to compile here instead of reaching a wizard with no
|
|
401
|
+
* question for it.
|
|
402
|
+
*/
|
|
403
|
+
export const ARM_PROOF_CHOICES: { readonly [K in ArmProof]: string } = {
|
|
404
|
+
challenge: "an authenticated Telegram challenge proves a human just approved arming",
|
|
405
|
+
"claim-only": "anyone who can invoke the already-privileged `omp-conductor arm` command can start dispatch once the live claim and poller pass",
|
|
406
|
+
};
|
|
407
|
+
|
|
384
408
|
export const RELEASE_REQUIREMENT_CHOICES: { readonly [K in ReleaseRequirement]: string } = {
|
|
385
|
-
"runs-settled": "every run
|
|
409
|
+
"runs-settled": "every run of the released repo actually merged, not merely reached a green PR",
|
|
410
|
+
"fleet-runs-settled": "every run in the project actually merged — suite-wide strictness for shapes that consume several repos",
|
|
386
411
|
"no-open-prs": "no pull request is still open against the branch being released",
|
|
387
412
|
"queue-drained": "nothing still carries the queue label",
|
|
388
413
|
"base-branch-green": "the newest observed post-merge base-branch workflows are green",
|
|
@@ -700,11 +725,33 @@ export async function createMissingLabels(trackerRepo: string, plan: LabelPlan[]
|
|
|
700
725
|
continue;
|
|
701
726
|
}
|
|
702
727
|
if (/already exists/i.test(r.stderr)) continue;
|
|
703
|
-
|
|
728
|
+
// The labels created before this one failed ride on the error: the setup
|
|
729
|
+
// transaction compensates the tracker exactly, and a throw without the
|
|
730
|
+
// partial list would orphan them (#652).
|
|
731
|
+
throw Object.assign(
|
|
732
|
+
new Error(`Could not create label "${label.name}" in ${trackerRepo}: ${briefly(r.stderr)}`),
|
|
733
|
+
{ created },
|
|
734
|
+
);
|
|
704
735
|
}
|
|
705
736
|
return created;
|
|
706
737
|
}
|
|
707
738
|
|
|
739
|
+
/**
|
|
740
|
+
* Compensation half of {@link createMissingLabels}: deletes exactly the labels
|
|
741
|
+
* a failed setup apply created, so the tracker returns to its pre-entry state.
|
|
742
|
+
* Runs only on rollback, where a deletion failure is a reported restoration
|
|
743
|
+
* failure — the orphaned label is named, never silently left behind (#652).
|
|
744
|
+
*/
|
|
745
|
+
export async function deleteCreatedLabels(trackerRepo: string, created: string[]): Promise<void> {
|
|
746
|
+
const failures: string[] = [];
|
|
747
|
+
for (const name of created) {
|
|
748
|
+
const r = await gh(["label", "delete", name, "--repo", trackerRepo, "--yes"]);
|
|
749
|
+
if (r.code === 0 || /not found/i.test(r.stderr)) continue;
|
|
750
|
+
failures.push(`could not delete label "${name}" in ${trackerRepo}: ${briefly(r.stderr)}`);
|
|
751
|
+
}
|
|
752
|
+
if (failures.length > 0) throw new Error(failures.join("; "));
|
|
753
|
+
}
|
|
754
|
+
|
|
708
755
|
const QUIET_INTERRUPT_ON: InterruptCategory[] = [
|
|
709
756
|
"tier2",
|
|
710
757
|
"fleet-stopped",
|
|
@@ -877,6 +924,10 @@ export function buildProject(a: SetupAnswers): ProjectConfig {
|
|
|
877
924
|
// Written out in full for the same reason: the file then says what a merge
|
|
878
925
|
// and a release require without anyone having to know a default (#129).
|
|
879
926
|
policy: clonePolicy(a.policy),
|
|
927
|
+
// Written out even when it is the default, so an operator amending the
|
|
928
|
+
// policy has a line in the file to point at — and the recovery playbook
|
|
929
|
+
// can read which proof the project opted into (#613).
|
|
930
|
+
arm: { proof: a.armProof },
|
|
880
931
|
...(a.recoveryMerges === undefined
|
|
881
932
|
? {}
|
|
882
933
|
: { recoveryMerges: a.recoveryMerges.map((entry) => ({ ...entry })) }),
|
|
@@ -964,6 +1015,9 @@ export function defaultAnswers(projectName: string, opts: { added?: boolean } =
|
|
|
964
1015
|
authority: { ...SETUP_DEFAULTS.authority },
|
|
965
1016
|
releaseGrants: { ...SETUP_DEFAULTS.releaseGrants },
|
|
966
1017
|
policy: clonePolicy(SETUP_DEFAULTS.policy),
|
|
1018
|
+
// The strict reading, matching the loader's absent-key default: a project
|
|
1019
|
+
// that never answered keeps today's authenticated challenge round-trip.
|
|
1020
|
+
armProof: DEFAULT_ARM_PROOF,
|
|
967
1021
|
orchestratorMode: SETUP_DEFAULTS.orchestratorMode,
|
|
968
1022
|
reportScope: SETUP_DEFAULT_REPORT_SCOPE,
|
|
969
1023
|
writeOrchestratorBrief: false,
|
|
@@ -1060,6 +1114,9 @@ export function answersFromProject(p: ProjectConfig): SetupAnswers {
|
|
|
1060
1114
|
authority: { ...p.authority },
|
|
1061
1115
|
releaseGrants: resolveReleaseGrants(p),
|
|
1062
1116
|
policy: resolvePolicy(p),
|
|
1117
|
+
// The loader materialises `arm` complete, so this is the answer the project
|
|
1118
|
+
// actually has — never a default guessed at the amend prompt.
|
|
1119
|
+
armProof: resolveArmProof(p),
|
|
1063
1120
|
orchestratorMode: p.escalation.orchestrator,
|
|
1064
1121
|
reportScope: reportScopeFromPolicy(p.reporting),
|
|
1065
1122
|
writeOrchestratorBrief: false,
|
package/src/status-render.ts
CHANGED
|
@@ -415,7 +415,17 @@ function formatProjectBody(
|
|
|
415
415
|
lines.push("active runs");
|
|
416
416
|
for (const r of s.activeRuns) {
|
|
417
417
|
const phase = workerPhases.get(r.issue);
|
|
418
|
-
|
|
418
|
+
// A run in a live review round reads `review-revision N`, distinct from
|
|
419
|
+
// a failure and from an ordinary continuation, with the round number
|
|
420
|
+
// from the durable revision row (#692). The live pause overlay still
|
|
421
|
+
// wins: an operator pause is the newer fact about the same session.
|
|
422
|
+
const round = s.reviewRounds?.[r.id];
|
|
423
|
+
const state =
|
|
424
|
+
phase === "pausing" || phase === "paused"
|
|
425
|
+
? phase
|
|
426
|
+
: round !== undefined
|
|
427
|
+
? `review-revision ${round}`
|
|
428
|
+
: r.state;
|
|
419
429
|
lines.push(
|
|
420
430
|
` #${r.issue} ${r.repo} ${state} attempt ${r.attempt} ` +
|
|
421
431
|
`${r.turns}/${r.maxTurns} turns ${r.spendUsd.toFixed(2)} ${r.branch}` +
|
package/src/store.ts
CHANGED
|
@@ -41,6 +41,7 @@ import type {
|
|
|
41
41
|
FrictionKind,
|
|
42
42
|
FrictionObservation,
|
|
43
43
|
FrictionSignal,
|
|
44
|
+
HistoricalInfraCandidate,
|
|
44
45
|
HeldNotice,
|
|
45
46
|
InterruptCategory,
|
|
46
47
|
HeldNoticeDraft,
|
|
@@ -60,6 +61,9 @@ import type {
|
|
|
60
61
|
ReportKind,
|
|
61
62
|
ReportRecord,
|
|
62
63
|
RecoveryAction,
|
|
64
|
+
ReviewReason,
|
|
65
|
+
ReviewRevisionOutcome,
|
|
66
|
+
ReviewRevisionRecord,
|
|
63
67
|
RunPatch,
|
|
64
68
|
RunRecord,
|
|
65
69
|
RunState,
|
|
@@ -99,6 +103,7 @@ const FRICTION_HOLD_REASONS: ReadonlySet<FrictionAdmissionReason> = new Set([
|
|
|
99
103
|
"failed-attempts",
|
|
100
104
|
"continuations",
|
|
101
105
|
"stale-base",
|
|
106
|
+
"critical-base-verify-error",
|
|
102
107
|
"parent-lookup-error",
|
|
103
108
|
"issue-state-lookup-error",
|
|
104
109
|
"open-pr-lookup-error",
|
|
@@ -361,6 +366,48 @@ interface MergeLockRow {
|
|
|
361
366
|
at: number;
|
|
362
367
|
}
|
|
363
368
|
|
|
369
|
+
/** The `review_revisions` table exactly as SQLite hands it back (#677). */
|
|
370
|
+
interface ReviewRevisionRow {
|
|
371
|
+
id: string;
|
|
372
|
+
project: string;
|
|
373
|
+
runId: string;
|
|
374
|
+
issue: number;
|
|
375
|
+
prUrl: string;
|
|
376
|
+
headSha: string;
|
|
377
|
+
findings: string;
|
|
378
|
+
round: number;
|
|
379
|
+
reason: string;
|
|
380
|
+
sessionFile: string | null;
|
|
381
|
+
requestedAt: number;
|
|
382
|
+
dispatchedAt: number | null;
|
|
383
|
+
settledAt: number | null;
|
|
384
|
+
outcome: string | null;
|
|
385
|
+
}
|
|
386
|
+
|
|
387
|
+
/**
|
|
388
|
+
* NULL columns become absent properties, matching the other row converters:
|
|
389
|
+
* a record read back out of the store deep-equals the one that went in.
|
|
390
|
+
*/
|
|
391
|
+
function toReviewRevision(row: ReviewRevisionRow): ReviewRevisionRecord {
|
|
392
|
+
const record: ReviewRevisionRecord = {
|
|
393
|
+
id: row.id,
|
|
394
|
+
project: row.project,
|
|
395
|
+
runId: row.runId,
|
|
396
|
+
issue: row.issue,
|
|
397
|
+
prUrl: row.prUrl,
|
|
398
|
+
headSha: row.headSha,
|
|
399
|
+
findings: row.findings,
|
|
400
|
+
round: row.round,
|
|
401
|
+
reason: row.reason as ReviewReason,
|
|
402
|
+
requestedAt: row.requestedAt,
|
|
403
|
+
};
|
|
404
|
+
if (row.sessionFile !== null) record.sessionFile = row.sessionFile;
|
|
405
|
+
if (row.dispatchedAt !== null) record.dispatchedAt = row.dispatchedAt;
|
|
406
|
+
if (row.settledAt !== null) record.settledAt = row.settledAt;
|
|
407
|
+
if (row.outcome !== null) record.outcome = row.outcome as ReviewRevisionOutcome;
|
|
408
|
+
return record;
|
|
409
|
+
}
|
|
410
|
+
|
|
364
411
|
/** The `intake_items` table exactly as SQLite hands it back (#299). */
|
|
365
412
|
interface IntakeRow {
|
|
366
413
|
id: string;
|
|
@@ -492,6 +539,22 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
492
539
|
CREATE INDEX IF NOT EXISTS runs_project_issue ON runs (project, issue);
|
|
493
540
|
CREATE INDEX IF NOT EXISTS runs_project_state ON runs (project, state);
|
|
494
541
|
|
|
542
|
+
-- The historical infrastructure reconciliation's per-project review cursor
|
|
543
|
+
-- (#638). A bounded pass examines settled ci-deterministic rows newest-first
|
|
544
|
+
-- and persists the boundary of the last row it definitively decided, so older
|
|
545
|
+
-- repairable rows are reached across later daemon starts instead of being
|
|
546
|
+
-- starved by repeated rescans of the newest non-matches. classifierVersion is
|
|
547
|
+
-- the signature-list fingerprint at the time the cursor advanced; when the
|
|
548
|
+
-- classifier learns a new signature the stamp goes stale and the pass restarts
|
|
549
|
+
-- from the newest row rather than skipping past newly recognised evidence.
|
|
550
|
+
CREATE TABLE IF NOT EXISTS historical_infra_review (
|
|
551
|
+
project TEXT PRIMARY KEY,
|
|
552
|
+
cursorStartedAt INTEGER NOT NULL,
|
|
553
|
+
cursorRowid INTEGER NOT NULL,
|
|
554
|
+
classifierVersion TEXT NOT NULL,
|
|
555
|
+
updatedAt INTEGER NOT NULL
|
|
556
|
+
);
|
|
557
|
+
|
|
495
558
|
CREATE TABLE IF NOT EXISTS base_health (
|
|
496
559
|
project TEXT NOT NULL,
|
|
497
560
|
repo TEXT NOT NULL,
|
|
@@ -693,6 +756,31 @@ CREATE TABLE IF NOT EXISTS merge_locks (
|
|
|
693
756
|
at INTEGER NOT NULL
|
|
694
757
|
);
|
|
695
758
|
|
|
759
|
+
-- One durable review-revision request (#677): an orchestrator returned a
|
|
760
|
+
-- green, run-owned pull request to its worker with blocking findings. Every
|
|
761
|
+
-- field the daemon needs to resume the exact session is persisted BEFORE the
|
|
762
|
+
-- worker is woken -- the findings, the exact reviewed head, the round number,
|
|
763
|
+
-- and the target run/session -- and the row doubles as the duplicate guard: a
|
|
764
|
+
-- revision whose row exists pending is one nobody may queue again.
|
|
765
|
+
CREATE TABLE IF NOT EXISTS review_revisions (
|
|
766
|
+
id TEXT PRIMARY KEY,
|
|
767
|
+
project TEXT NOT NULL,
|
|
768
|
+
runId TEXT NOT NULL,
|
|
769
|
+
issue INTEGER NOT NULL,
|
|
770
|
+
prUrl TEXT NOT NULL,
|
|
771
|
+
headSha TEXT NOT NULL,
|
|
772
|
+
findings TEXT NOT NULL,
|
|
773
|
+
round INTEGER NOT NULL,
|
|
774
|
+
reason TEXT NOT NULL,
|
|
775
|
+
sessionFile TEXT,
|
|
776
|
+
requestedAt INTEGER NOT NULL,
|
|
777
|
+
dispatchedAt INTEGER,
|
|
778
|
+
settledAt INTEGER,
|
|
779
|
+
outcome TEXT
|
|
780
|
+
);
|
|
781
|
+
CREATE INDEX IF NOT EXISTS review_revisions_pending
|
|
782
|
+
ON review_revisions (project, dispatchedAt, runId);
|
|
783
|
+
|
|
696
784
|
-- Questions the orchestrator has put to its operator, and their answers (#136).
|
|
697
785
|
--
|
|
698
786
|
-- The reason this is a table and not the session's memory: a question lived only
|
|
@@ -1143,12 +1231,59 @@ export function dbPath(): string {
|
|
|
1143
1231
|
return join(stateDir(), "conductor.db");
|
|
1144
1232
|
}
|
|
1145
1233
|
|
|
1234
|
+
/**
|
|
1235
|
+
* Live worker count across the run store, opening the database WITHOUT
|
|
1236
|
+
* creating or upgrading it. The setup apply's quiescence preflight must count
|
|
1237
|
+
* workers before the acknowledged barrier, and database preparation is itself
|
|
1238
|
+
* one of the mutations that barrier must precede — so the observation that
|
|
1239
|
+
* gates it cannot be the thing that creates or migrates the store (#618). The
|
|
1240
|
+
* on-disk database is opened read-only (never created, never "prepared"); a
|
|
1241
|
+
* missing file reports zero because no run has ever been persisted, which the
|
|
1242
|
+
* caller only trusts when no daemon is running either. An unreadable or
|
|
1243
|
+
* schema-broken file throws rather than reading as zero — fail closed, never
|
|
1244
|
+
* absence.
|
|
1245
|
+
*/
|
|
1246
|
+
export function liveWorkersReadOnly(project?: string): number {
|
|
1247
|
+
const path = dbPath();
|
|
1248
|
+
if (!existsSync(path)) return 0;
|
|
1249
|
+
let db: Database | undefined;
|
|
1250
|
+
try {
|
|
1251
|
+
db = new Database(path, { create: false, readonly: true });
|
|
1252
|
+
db.exec("PRAGMA busy_timeout = 5000;");
|
|
1253
|
+
const placeholders = LIVE_STATES.map(() => "?").join(", ");
|
|
1254
|
+
// `undefined` is the host-global selector: every project's live rows count,
|
|
1255
|
+
// never only the rows of a project literally named "" — a host-global
|
|
1256
|
+
// observation that scoped to `project = ''` hid every real named row and
|
|
1257
|
+
// read zero beside a running fleet (#651 review #3).
|
|
1258
|
+
const row =
|
|
1259
|
+
project === undefined
|
|
1260
|
+
? db
|
|
1261
|
+
.query<{ n: number }, SqlValue[]>(
|
|
1262
|
+
`SELECT COUNT(*) AS n FROM runs WHERE state IN (${placeholders})`,
|
|
1263
|
+
)
|
|
1264
|
+
.get(...LIVE_STATES)
|
|
1265
|
+
: db
|
|
1266
|
+
.query<{ n: number }, SqlValue[]>(
|
|
1267
|
+
`SELECT COUNT(*) AS n FROM runs WHERE project = ? AND state IN (${placeholders})`,
|
|
1268
|
+
)
|
|
1269
|
+
.get(project, ...LIVE_STATES);
|
|
1270
|
+
return row?.n ?? 0;
|
|
1271
|
+
} catch (err) {
|
|
1272
|
+
throw new Error(
|
|
1273
|
+
`cannot read ${path} to prove the fleet is quiescent: ${err instanceof Error ? err.message : String(err)}`,
|
|
1274
|
+
);
|
|
1275
|
+
} finally {
|
|
1276
|
+
db?.close();
|
|
1277
|
+
}
|
|
1278
|
+
}
|
|
1279
|
+
|
|
1146
1280
|
/** The stem every restorable `conductor.db` snapshot is published under. */
|
|
1147
1281
|
export const DB_SNAPSHOT_STEM = "conductor.db.bak";
|
|
1148
1282
|
|
|
1149
1283
|
/**
|
|
1150
|
-
* A
|
|
1151
|
-
*
|
|
1284
|
+
* A single-file, transaction-consistent copy of the store at `source` into
|
|
1285
|
+
* `target` — the shared core of {@link snapshotDb} and the setup apply's
|
|
1286
|
+
* rollback capture.
|
|
1152
1287
|
*
|
|
1153
1288
|
* Uses SQLite's own `VACUUM INTO`, never a plain file copy: the store opens
|
|
1154
1289
|
* with `journal_mode=WAL`, and a `copyFileSync` can capture a torn database
|
|
@@ -1158,10 +1293,30 @@ export const DB_SNAPSHOT_STEM = "conductor.db.bak";
|
|
|
1158
1293
|
* writes a single-file snapshot, so the result opens cleanly in a fresh
|
|
1159
1294
|
* connection and contains every committed row.
|
|
1160
1295
|
*
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1163
|
-
*
|
|
1296
|
+
* Runs against its own read of the source, deliberately not the caller's
|
|
1297
|
+
* writer connection, so a snapshot can be taken while a writer holds an
|
|
1298
|
+
* uncheckpointed WAL. `:memory:` has no on-disk state to snapshot and is
|
|
1164
1299
|
* refused.
|
|
1300
|
+
*/
|
|
1301
|
+
export function vacuumInto(source: string, target: string): void {
|
|
1302
|
+
if (source === ":memory:") throw new Error("cannot snapshot an in-memory store");
|
|
1303
|
+
if (!existsSync(source)) {
|
|
1304
|
+
throw new Error(`cannot snapshot ${source}: the store does not exist`);
|
|
1305
|
+
}
|
|
1306
|
+
let connection: Database | undefined;
|
|
1307
|
+
try {
|
|
1308
|
+
connection = new Database(source);
|
|
1309
|
+
connection.exec("PRAGMA busy_timeout = 5000;");
|
|
1310
|
+
// `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
|
|
1311
|
+
connection.exec(`VACUUM INTO '${target.replaceAll("'", "''")}'`);
|
|
1312
|
+
} finally {
|
|
1313
|
+
connection?.close();
|
|
1314
|
+
}
|
|
1315
|
+
}
|
|
1316
|
+
|
|
1317
|
+
/**
|
|
1318
|
+
* A fresh, restorable copy of the store at `source`, published under `destDir`
|
|
1319
|
+
* with the same temp-then-atomic-publish convention as the config backups.
|
|
1165
1320
|
*
|
|
1166
1321
|
* Returns the published snapshot path.
|
|
1167
1322
|
*/
|
|
@@ -1172,18 +1327,12 @@ export function snapshotDb(source: string, destDir: string): string {
|
|
|
1172
1327
|
}
|
|
1173
1328
|
mkdirSync(destDir, { recursive: true });
|
|
1174
1329
|
const temporary = join(destDir, `.${DB_SNAPSHOT_STEM}-${process.pid}.${randomUUID()}.tmp`);
|
|
1175
|
-
let connection: Database | undefined;
|
|
1176
1330
|
try {
|
|
1177
|
-
|
|
1178
|
-
connection.exec("PRAGMA busy_timeout = 5000;");
|
|
1179
|
-
// `VACUUM INTO` takes the target as an SQL literal, not a bound parameter.
|
|
1180
|
-
connection.exec(`VACUUM INTO '${temporary.replaceAll("'", "''")}'`);
|
|
1331
|
+
vacuumInto(source, temporary);
|
|
1181
1332
|
} catch (err) {
|
|
1182
1333
|
// Never leave a half-written snapshot on disk.
|
|
1183
1334
|
rmSync(temporary, { force: true });
|
|
1184
1335
|
throw err;
|
|
1185
|
-
} finally {
|
|
1186
|
-
connection?.close();
|
|
1187
1336
|
}
|
|
1188
1337
|
return publishTempFile(temporary, destDir, `${DB_SNAPSHOT_STEM}-${backupTimestamp()}`);
|
|
1189
1338
|
}
|
|
@@ -1644,6 +1793,56 @@ export function openStore(dbPath: string): Store {
|
|
|
1644
1793
|
ORDER BY startedAt DESC, rowid DESC
|
|
1645
1794
|
LIMIT ?`,
|
|
1646
1795
|
);
|
|
1796
|
+
// The settled rows a classifier correction can no longer reach (#638). A
|
|
1797
|
+
// `ci-deterministic`/`escalate` row never re-enters `selectUnclassified`
|
|
1798
|
+
// (its class is set and its recovery is not retryable), so evidence the
|
|
1799
|
+
// classifier learned after the row was written — the codeload setup 429 of
|
|
1800
|
+
// #177 — stays a charged attempt forever. This WHERE is the historical
|
|
1801
|
+
// reconciliation's exact predicate: nothing but rows currently reading
|
|
1802
|
+
// `ci-deterministic`, never a class a human re-read, and the class itself is
|
|
1803
|
+
// only ever produced for `state = 'failed'`. Bounded like every other sweep
|
|
1804
|
+
// read, because each candidate costs tracker calls to re-fetch its evidence.
|
|
1805
|
+
// `after` is the persisted review cursor: candidates are only rows strictly
|
|
1806
|
+
// older than it, so a bounded pass progresses through a long misclassified
|
|
1807
|
+
// history instead of rescanning the newest non-matches forever. A NULL cursor
|
|
1808
|
+
// (first pass, or one rendered stale by a classifier-signature change) means
|
|
1809
|
+
// "start from the newest". `SELECT rowid, *` exposes the hidden tie-breaker
|
|
1810
|
+
// so the pass can record an exact resume boundary.
|
|
1811
|
+
const selectHistoricalInfraCandidates = db.query<RunRow & { rowid: number }, SqlValue[]>(
|
|
1812
|
+
`SELECT rowid, * FROM runs
|
|
1813
|
+
WHERE project = ? AND failureClass = 'ci-deterministic' AND state = 'failed'
|
|
1814
|
+
AND (? IS NULL OR (startedAt < ? OR (startedAt = ? AND rowid < ?)))
|
|
1815
|
+
ORDER BY startedAt DESC, rowid DESC
|
|
1816
|
+
LIMIT ?`,
|
|
1817
|
+
);
|
|
1818
|
+
const selectHistoricalInfraCursor = db.query<
|
|
1819
|
+
{ cursorStartedAt: number; cursorRowid: number; classifierVersion: string },
|
|
1820
|
+
[string]
|
|
1821
|
+
>(
|
|
1822
|
+
`SELECT cursorStartedAt, cursorRowid, classifierVersion FROM historical_infra_review
|
|
1823
|
+
WHERE project = ?`,
|
|
1824
|
+
);
|
|
1825
|
+
const upsertHistoricalInfraCursor = db.query<
|
|
1826
|
+
unknown,
|
|
1827
|
+
[string, number, number, string, number]
|
|
1828
|
+
>(
|
|
1829
|
+
`INSERT INTO historical_infra_review
|
|
1830
|
+
(project, cursorStartedAt, cursorRowid, classifierVersion, updatedAt)
|
|
1831
|
+
VALUES (?, ?, ?, ?, ?)
|
|
1832
|
+
ON CONFLICT(project) DO UPDATE SET
|
|
1833
|
+
cursorStartedAt = excluded.cursorStartedAt,
|
|
1834
|
+
cursorRowid = excluded.cursorRowid,
|
|
1835
|
+
classifierVersion = excluded.classifierVersion,
|
|
1836
|
+
updatedAt = excluded.updatedAt`,
|
|
1837
|
+
);
|
|
1838
|
+
// The one mutation the historical reconciliation may make. Guarded by the
|
|
1839
|
+
// current class, so a second pass is a no-op and a race can never reclassify
|
|
1840
|
+
// a row a human has re-read. Deliberately only `failureClass` — never
|
|
1841
|
+
// `recoveryAction` or `recoveredAt`: relieving the budget is the point,
|
|
1842
|
+
// re-animating a months-old run into the requeue sweep is not (#638).
|
|
1843
|
+
const reclassifyToInfra = db.query<unknown, [string]>(
|
|
1844
|
+
`UPDATE runs SET failureClass = 'ci-infra' WHERE id = ? AND failureClass = 'ci-deterministic'`,
|
|
1845
|
+
);
|
|
1647
1846
|
const selectFailureClassCounts = db.query<{ cls: string; n: number }, [string]>(
|
|
1648
1847
|
`SELECT failureClass AS cls, COUNT(*) AS n FROM runs
|
|
1649
1848
|
WHERE project = ? AND failureClass IS NOT NULL AND recoveredAt IS NULL
|
|
@@ -2319,6 +2518,72 @@ export function openStore(dbPath: string): Store {
|
|
|
2319
2518
|
},
|
|
2320
2519
|
);
|
|
2321
2520
|
|
|
2521
|
+
// The durable review-revision request outbox (#677): the orchestrator's verb
|
|
2522
|
+
// writes the row (findings, head, round, target session) and the daemon's
|
|
2523
|
+
// next dispatch pass wakes it. The duplicate in-flight guard is the insert:
|
|
2524
|
+
// a pending row for the same run refuses the second request atomically, and
|
|
2525
|
+
// once dispatched the run row's own state (pushed-green → running) is the
|
|
2526
|
+
// in-flight marker.
|
|
2527
|
+
const insertReviewRevision = db.query<unknown, SqlValue[]>(
|
|
2528
|
+
`INSERT INTO review_revisions
|
|
2529
|
+
(id, project, runId, issue, prUrl, headSha, findings, round, reason, sessionFile, requestedAt, dispatchedAt, settledAt, outcome)
|
|
2530
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, NULL, NULL, NULL)`,
|
|
2531
|
+
);
|
|
2532
|
+
const selectPendingReviewRevisions = db.query<ReviewRevisionRow, [string]>(
|
|
2533
|
+
`SELECT * FROM review_revisions
|
|
2534
|
+
WHERE project = ? AND dispatchedAt IS NULL AND settledAt IS NULL
|
|
2535
|
+
ORDER BY requestedAt ASC, rowid ASC`,
|
|
2536
|
+
);
|
|
2537
|
+
const selectPendingReviewForRun = db.query<ReviewRevisionRow, [string, string]>(
|
|
2538
|
+
`SELECT * FROM review_revisions
|
|
2539
|
+
WHERE project = ? AND runId = ? AND dispatchedAt IS NULL AND settledAt IS NULL
|
|
2540
|
+
LIMIT 1`,
|
|
2541
|
+
);
|
|
2542
|
+
const selectLatestReviewRound = db.query<{ n: number | null }, [string, string]>(
|
|
2543
|
+
`SELECT MAX(round) AS n FROM review_revisions WHERE project = ? AND runId = ?`,
|
|
2544
|
+
);
|
|
2545
|
+
const markReviewRevisionDispatchedRow = db.query<unknown, [number, string]>(
|
|
2546
|
+
`UPDATE review_revisions SET dispatchedAt = ? WHERE id = ?`,
|
|
2547
|
+
);
|
|
2548
|
+
const settleReviewRevisionRow = db.query<unknown, [string, number, string]>(
|
|
2549
|
+
`UPDATE review_revisions SET outcome = ?, settledAt = ? WHERE id = ?`,
|
|
2550
|
+
);
|
|
2551
|
+
const selectUnsettledReviewRevisions = db.query<ReviewRevisionRow, [string]>(
|
|
2552
|
+
`SELECT * FROM review_revisions
|
|
2553
|
+
WHERE project = ? AND settledAt IS NULL
|
|
2554
|
+
ORDER BY requestedAt ASC, rowid ASC`,
|
|
2555
|
+
);
|
|
2556
|
+
const requeueReviewRevisionRow = db.query<unknown, [string]>(
|
|
2557
|
+
`UPDATE review_revisions SET dispatchedAt = NULL WHERE id = ?`,
|
|
2558
|
+
);
|
|
2559
|
+
const claimRunForReviewRow = db.query<unknown, [string]>(
|
|
2560
|
+
`UPDATE runs SET state = 'running', endedAt = NULL WHERE id = ? AND state = 'pushed-green'`,
|
|
2561
|
+
);
|
|
2562
|
+
|
|
2563
|
+
// The claim and the insert share a transaction so two concurrent revision
|
|
2564
|
+
// requests for one run cannot both win: the second sees the first's pending
|
|
2565
|
+
// row and returns undefined, which the verb turns into the in-flight refusal.
|
|
2566
|
+
const claimReviewRevision = db.transaction(
|
|
2567
|
+
(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined => {
|
|
2568
|
+
if (selectPendingReviewForRun.get(draft.project, draft.runId) !== null) return undefined;
|
|
2569
|
+
const record: ReviewRevisionRecord = { ...draft, id: crypto.randomUUID() };
|
|
2570
|
+
insertReviewRevision.run(
|
|
2571
|
+
record.id,
|
|
2572
|
+
record.project,
|
|
2573
|
+
record.runId,
|
|
2574
|
+
record.issue,
|
|
2575
|
+
record.prUrl,
|
|
2576
|
+
record.headSha,
|
|
2577
|
+
record.findings,
|
|
2578
|
+
record.round,
|
|
2579
|
+
record.reason,
|
|
2580
|
+
record.sessionFile ?? null,
|
|
2581
|
+
record.requestedAt,
|
|
2582
|
+
);
|
|
2583
|
+
return record;
|
|
2584
|
+
},
|
|
2585
|
+
);
|
|
2586
|
+
|
|
2322
2587
|
// The label projection outbox (#201). A decided label change is a row until
|
|
2323
2588
|
// the tracker has applied it; the projector drains in id order, and one
|
|
2324
2589
|
// issue's ops are atomic as a run — a failed op parks the rest of its issue
|
|
@@ -2526,6 +2791,30 @@ export function openStore(dbPath: string): Store {
|
|
|
2526
2791
|
runsForProjectPr(project: string, prUrl: string, mergedSinceEpochMs: number): RunRecord[] {
|
|
2527
2792
|
return selectRunsForPr.all(project, prUrl, mergedSinceEpochMs).map(toRecord);
|
|
2528
2793
|
},
|
|
2794
|
+
createReviewRevision(draft: Omit<ReviewRevisionRecord, "id">): ReviewRevisionRecord | undefined {
|
|
2795
|
+
return claimReviewRevision(draft);
|
|
2796
|
+
},
|
|
2797
|
+
pendingReviewRevisions(project: string): ReviewRevisionRecord[] {
|
|
2798
|
+
return selectPendingReviewRevisions.all(project).map(toReviewRevision);
|
|
2799
|
+
},
|
|
2800
|
+
latestReviewRound(project: string, runId: string): number {
|
|
2801
|
+
return selectLatestReviewRound.get(project, runId)?.n ?? 0;
|
|
2802
|
+
},
|
|
2803
|
+
markReviewRevisionDispatched(id: string, at: number): void {
|
|
2804
|
+
markReviewRevisionDispatchedRow.run(at, id);
|
|
2805
|
+
},
|
|
2806
|
+
settleReviewRevision(id: string, outcome: ReviewRevisionOutcome, at: number): void {
|
|
2807
|
+
settleReviewRevisionRow.run(outcome, at, id);
|
|
2808
|
+
},
|
|
2809
|
+
unsettledReviewRevisions(project: string): ReviewRevisionRecord[] {
|
|
2810
|
+
return selectUnsettledReviewRevisions.all(project).map(toReviewRevision);
|
|
2811
|
+
},
|
|
2812
|
+
requeueReviewRevision(id: string): void {
|
|
2813
|
+
requeueReviewRevisionRow.run(id);
|
|
2814
|
+
},
|
|
2815
|
+
claimRunForReview(runId: string): boolean {
|
|
2816
|
+
return claimRunForReviewRow.run(runId).changes > 0;
|
|
2817
|
+
},
|
|
2529
2818
|
runsNeedingBaseCheck(project: string, limit = 20): RunRecord[] {
|
|
2530
2819
|
return selectPendingBaseChecks.all(project, limit).map(toRecord);
|
|
2531
2820
|
},
|
|
@@ -3066,6 +3355,38 @@ export function openStore(dbPath: string): Store {
|
|
|
3066
3355
|
return selectUnclassified.all(project, limit).map(toRecord);
|
|
3067
3356
|
},
|
|
3068
3357
|
|
|
3358
|
+
historicalInfraCandidates(
|
|
3359
|
+
project: string,
|
|
3360
|
+
limit = 20,
|
|
3361
|
+
after?: { startedAt: number; rowid: number },
|
|
3362
|
+
): HistoricalInfraCandidate[] {
|
|
3363
|
+
const cursorAt = after?.startedAt ?? null;
|
|
3364
|
+
const cursorRowid = after?.rowid ?? null;
|
|
3365
|
+
return selectHistoricalInfraCandidates
|
|
3366
|
+
.all(project, cursorAt, cursorAt, cursorAt, cursorRowid, limit)
|
|
3367
|
+
.map((row) => ({ ...toRecord(row), rowid: row.rowid }));
|
|
3368
|
+
},
|
|
3369
|
+
|
|
3370
|
+
historicalInfraCursor(project: string) {
|
|
3371
|
+
const row = selectHistoricalInfraCursor.get(project);
|
|
3372
|
+
return row == null
|
|
3373
|
+
? undefined
|
|
3374
|
+
: { startedAt: row.cursorStartedAt, rowid: row.cursorRowid, classifierVersion: row.classifierVersion };
|
|
3375
|
+
},
|
|
3376
|
+
|
|
3377
|
+
setHistoricalInfraCursor(
|
|
3378
|
+
project: string,
|
|
3379
|
+
startedAt: number,
|
|
3380
|
+
rowid: number,
|
|
3381
|
+
classifierVersion: string,
|
|
3382
|
+
): void {
|
|
3383
|
+
upsertHistoricalInfraCursor.run(project, startedAt, rowid, classifierVersion, Date.now());
|
|
3384
|
+
},
|
|
3385
|
+
|
|
3386
|
+
reclassifyInfra(id: string): boolean {
|
|
3387
|
+
return reclassifyToInfra.run(id).changes > 0;
|
|
3388
|
+
},
|
|
3389
|
+
|
|
3069
3390
|
failureClassCounts(project: string): { cls: FailureClass; n: number }[] {
|
|
3070
3391
|
return selectFailureClassCounts
|
|
3071
3392
|
.all(project)
|