omp-conductor 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -21
- package/package.json +1 -1
- package/src/backups.ts +46 -0
- package/src/board.ts +254 -38
- package/src/brief-upgrade.ts +1 -30
- package/src/cli.ts +17 -5
- package/src/config.ts +13 -0
- package/src/daemon.ts +238 -123
- package/src/diff-flags.ts +35 -6
- package/src/fleet.ts +28 -2
- package/src/label-projection.ts +93 -0
- package/src/routing.ts +20 -0
- package/src/store.ts +254 -2
- package/src/tracker/github.ts +266 -18
- package/src/types.ts +75 -3
- package/src/unblock.ts +70 -19
- package/src/upgrade.ts +28 -4
package/src/diff-flags.ts
CHANGED
|
@@ -435,6 +435,14 @@ function evidence(text: string): string {
|
|
|
435
435
|
export interface SettlementAudit {
|
|
436
436
|
/** The worker's final report, verbatim. */
|
|
437
437
|
report: string;
|
|
438
|
+
/** Earlier attempts' reports for the same issue, oldest first. Their
|
|
439
|
+
* `changed:` disclosures are pooled as coverage so a file disclosed in a
|
|
440
|
+
* prior attempt is never flagged as undisclosed in the final one, while the
|
|
441
|
+
* reverse (unmatched-claim) direction reads only the current report — a file
|
|
442
|
+
* claimed in attempt 1 and reverted by attempt 3 must not come back as a new
|
|
443
|
+
* finding (#199). Absent means this is the only attempt, or older rows never
|
|
444
|
+
* stored a report. */
|
|
445
|
+
priorReports?: readonly string[];
|
|
438
446
|
/** The dispatching issue's title and body — the attribution source. */
|
|
439
447
|
issueText: string;
|
|
440
448
|
diff: PrDiff;
|
|
@@ -457,8 +465,13 @@ export function analyseSettlement(audit: SettlementAudit): SettlementFlag[] {
|
|
|
457
465
|
function reconcileChanged(audit: SettlementAudit, flags: SettlementFlag[]): void {
|
|
458
466
|
const touched = audit.diff.files.filter((f) => DERIVED_FILE[basename(f.path)] !== true);
|
|
459
467
|
const claims = claimedPaths(audit.report);
|
|
468
|
+
const priorClaims = claimedPaths((audit.priorReports ?? []).join("\n"));
|
|
469
|
+
// Coverage pools the current report with every prior attempt's disclosures:
|
|
470
|
+
// a file the final report no longer names was disclosed while the work was
|
|
471
|
+
// still in flight, so it must not be flagged undisclosed (#199).
|
|
472
|
+
const coverage = [...new Set([...claims, ...priorClaims])];
|
|
460
473
|
|
|
461
|
-
if (claims.length === 0) {
|
|
474
|
+
if (claims.length === 0 && priorClaims.length === 0) {
|
|
462
475
|
// The degenerate case of the same check: with no usable `changed:` line
|
|
463
476
|
// every file is undisclosed, and saying so once beats saying it per file.
|
|
464
477
|
// Reporting it at all is what stops the check being defeated by writing
|
|
@@ -476,8 +489,8 @@ function reconcileChanged(audit: SettlementAudit, flags: SettlementFlag[]): void
|
|
|
476
489
|
}
|
|
477
490
|
|
|
478
491
|
for (const file of touched) {
|
|
479
|
-
if (
|
|
480
|
-
if (file.previousPath !== undefined &&
|
|
492
|
+
if (coverage.some((claim) => covers(claim, file.path))) continue;
|
|
493
|
+
if (file.previousPath !== undefined && coverage.some((claim) => covers(claim, file.previousPath ?? ""))) {
|
|
481
494
|
continue;
|
|
482
495
|
}
|
|
483
496
|
flags.push({
|
|
@@ -491,6 +504,10 @@ function reconcileChanged(audit: SettlementAudit, flags: SettlementFlag[]): void
|
|
|
491
504
|
// file it edited and then reverted, or one it renamed away from. Worth
|
|
492
505
|
// surfacing because a `changed:` line that describes a different PR is the
|
|
493
506
|
// signature of a report written from memory rather than from `git diff`.
|
|
507
|
+
//
|
|
508
|
+
// It iterates only the CURRENT report's claims (#199). A file claimed in
|
|
509
|
+
// attempt 1 and reverted by attempt 3 was true then and false now; pooling
|
|
510
|
+
// priors into this loop would turn every honest revert into a fresh finding.
|
|
494
511
|
for (const claim of claims) {
|
|
495
512
|
if (audit.diff.files.some((f) => covers(claim, f.path) || covers(claim, f.previousPath ?? ""))) {
|
|
496
513
|
continue;
|
|
@@ -661,18 +678,30 @@ const HEADING = "settlement audit";
|
|
|
661
678
|
* that could not be read in full — then the absence of flags is not evidence of
|
|
662
679
|
* anything, and saying so is the honest report.
|
|
663
680
|
*/
|
|
681
|
+
/** Appended to the heading when a multi-attempt audit pooled prior disclosures,
|
|
682
|
+
* so a reader is told the coverage is wider than this one report (#199). */
|
|
683
|
+
function pooledHeading(attempts: number | undefined): string {
|
|
684
|
+
return attempts !== undefined && attempts > 1
|
|
685
|
+
? ` — disclosures pooled across ${attempts} attempt(s)`
|
|
686
|
+
: "";
|
|
687
|
+
}
|
|
688
|
+
|
|
664
689
|
export function formatSettlementFlags(
|
|
665
690
|
flags: readonly SettlementFlag[],
|
|
666
|
-
diff: { truncated: boolean } = { truncated: false },
|
|
691
|
+
diff: { truncated: boolean; attempts?: number } = { truncated: false },
|
|
667
692
|
): string[] {
|
|
668
693
|
if (flags.length === 0) {
|
|
669
694
|
return diff.truncated
|
|
670
|
-
? [
|
|
695
|
+
? [
|
|
696
|
+
`${HEADING}: no flags, but the PR diff was too large to read in full` +
|
|
697
|
+
`${pooledHeading(diff.attempts)} — this is not a clean bill`,
|
|
698
|
+
]
|
|
671
699
|
: [];
|
|
672
700
|
}
|
|
673
701
|
const lines = [
|
|
674
702
|
`${HEADING}: ${flags.length} advisory flag(s) — the run's state is unchanged by them` +
|
|
675
|
-
(diff.truncated ? ", and the PR diff was too large to read in full" : "")
|
|
703
|
+
(diff.truncated ? ", and the PR diff was too large to read in full" : "") +
|
|
704
|
+
pooledHeading(diff.attempts),
|
|
676
705
|
];
|
|
677
706
|
for (const flag of flags.slice(0, RENDERED_FLAGS)) {
|
|
678
707
|
lines.push(
|
package/src/fleet.ts
CHANGED
|
@@ -1076,11 +1076,37 @@ function formatProjectBody(s: StatusSnapshot): string {
|
|
|
1076
1076
|
// fleet (#110).
|
|
1077
1077
|
` plan usage ${planUsageLine(s.planUsage)}`,
|
|
1078
1078
|
// The tracker's API budget, when the renderer could read it. Absent on a
|
|
1079
|
-
// broken `gh`: one missing row, never a broken report (#188).
|
|
1079
|
+
// broken `gh`: one missing row, never a broken report (#188). Next to it,
|
|
1080
|
+
// the refusals the tracker actually observed, so a budget that looks
|
|
1081
|
+
// healthy next to every call being refused is still visible (#198).
|
|
1080
1082
|
...(s.github === undefined
|
|
1081
1083
|
? []
|
|
1082
1084
|
: [
|
|
1083
|
-
` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)
|
|
1085
|
+
` github graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)` +
|
|
1086
|
+
(s.ghRefusals === undefined || s.ghRefusals.count === 0
|
|
1087
|
+
? ""
|
|
1088
|
+
: ` — ${s.ghRefusals.count} refusal(s) in last 5m (last ${new Date(s.ghRefusals.latestAt ?? Date.now()).toISOString().slice(11, 19)}Z)`),
|
|
1089
|
+
]),
|
|
1090
|
+
// The daemon's own observed github traffic today (#198): a separate row
|
|
1091
|
+
// from the polled budget above, so one missing `status` read never hides
|
|
1092
|
+
// the other. `onCall` counts every spawn including ones that end 304, so
|
|
1093
|
+
// `daemon` is spawns and `daemon-304` is the unbilled subset — billed ≈
|
|
1094
|
+
// difference, and the polled budget row stays the authority (#203).
|
|
1095
|
+
` github calls daemon ${s.ghCallsToday?.find((c) => c.source === "daemon")?.calls ?? 0} today` +
|
|
1096
|
+
((s.ghCallsToday?.find((c) => c.source === "daemon-304")?.calls ?? 0) === 0
|
|
1097
|
+
? ""
|
|
1098
|
+
: ` (${s.ghCallsToday!.find((c) => c.source === "daemon-304")!.calls} free 304s)`),
|
|
1099
|
+
// Label-projection ops the tracker has not applied yet (#201): GitHub is
|
|
1100
|
+
// behind what the store decided, and the operator can see the lag instead
|
|
1101
|
+
// of discovering it as a stale label or a missing one.
|
|
1102
|
+
...(s.labelOps === undefined
|
|
1103
|
+
? []
|
|
1104
|
+
: [
|
|
1105
|
+
` labels projection ${s.labelOps.pending} pending (oldest ${
|
|
1106
|
+
s.labelOps.oldestAgeMs >= 60_000
|
|
1107
|
+
? `${Math.round(s.labelOps.oldestAgeMs / 60_000)}m`
|
|
1108
|
+
: `${Math.max(1, Math.round(s.labelOps.oldestAgeMs / 1000))}s`
|
|
1109
|
+
})`,
|
|
1084
1110
|
]),
|
|
1085
1111
|
` new worker turns ${s.caps.workerMaxTurns}`,
|
|
1086
1112
|
` worker wall clock ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The label projection outbox (#201).
|
|
3
|
+
*
|
|
4
|
+
* The dispatcher used to write GitHub labels directly, which made the label —
|
|
5
|
+
* not the store — the crash guard against double dispatch, and let one refused
|
|
6
|
+
* label write strand an issue (`#184`, `#198`). Dispatch state is now
|
|
7
|
+
* store-authoritative; every decided label change is an outbox row that this
|
|
8
|
+
* module projects onto the tracker with retry.
|
|
9
|
+
*
|
|
10
|
+
* Both the daemon tick and the `unblock` CLI path drain through the same
|
|
11
|
+
* `projectLabels` so a 403 no longer takes either down: the op defers with
|
|
12
|
+
* backoff and the daemon retries it. Per-issue ops apply in id order, which is
|
|
13
|
+
* what makes a swap atomic — `add` lands before the paired `remove` exactly as
|
|
14
|
+
* enqueued.
|
|
15
|
+
*/
|
|
16
|
+
|
|
17
|
+
import { GhRateLimitError, isRateLimitRefusal } from "./tracker/github.ts";
|
|
18
|
+
import type { ProjectConfig, Store, Tracker } from "./types.ts";
|
|
19
|
+
|
|
20
|
+
/** Retry backoff for one failed label op: 30s doubling, capped at 15m (#201). */
|
|
21
|
+
export function labelOpBackoffMs(attempts: number): number {
|
|
22
|
+
return Math.min(30_000 * 2 ** attempts, 15 * 60_000);
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function errText(e: unknown): string {
|
|
26
|
+
return e instanceof Error ? e.message : String(e);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Drain as many due label ops as the tracker will take, in id order.
|
|
31
|
+
*
|
|
32
|
+
* A failed op defers with backoff and holds the rest of its issue for the next
|
|
33
|
+
* pass: a later op for the same issue must not land while an earlier one could
|
|
34
|
+
* not (that is the atomicity `#184` needed — add-queue-label must not beat
|
|
35
|
+
* remove-blocked when they were enqueued in that order). Other issues continue.
|
|
36
|
+
*
|
|
37
|
+
* Returns what happened so `unblock` can say "queued, the daemon retries"
|
|
38
|
+
* instead of pretending a deferred op was applied.
|
|
39
|
+
*/
|
|
40
|
+
export async function projectLabels(
|
|
41
|
+
store: Store,
|
|
42
|
+
tracker: Tracker,
|
|
43
|
+
project: ProjectConfig,
|
|
44
|
+
): Promise<{ applied: number; deferred: number }> {
|
|
45
|
+
let applied = 0;
|
|
46
|
+
let deferred = 0;
|
|
47
|
+
// `pendingLabelOps` returns at most the oldest owed row per issue (it
|
|
48
|
+
// excludes any row that still has an earlier pending sibling), so a swap's
|
|
49
|
+
// second op can never land before its first — per-issue atomicity enforced
|
|
50
|
+
// across calls and processes, not just one pass. Re-querying lets a healthy
|
|
51
|
+
// chain drain fully in this one call (unblock clears every state label in
|
|
52
|
+
// one go), while a failing op defers its issue and parks the rest until it
|
|
53
|
+
// settles.
|
|
54
|
+
for (;;) {
|
|
55
|
+
const due = store.pendingLabelOps(project.name, Date.now());
|
|
56
|
+
if (due.length === 0) break;
|
|
57
|
+
let progressed = false;
|
|
58
|
+
for (const op of due) {
|
|
59
|
+
try {
|
|
60
|
+
if (op.op === "add") await tracker.addLabel(op.issue, op.label);
|
|
61
|
+
else await tracker.removeLabel(op.issue, op.label);
|
|
62
|
+
store.settleLabelOp(op.id);
|
|
63
|
+
applied += 1;
|
|
64
|
+
progressed = true;
|
|
65
|
+
} catch (err) {
|
|
66
|
+
// A rate-limit refusal defers WITHOUT counting an attempt: a shared
|
|
67
|
+
// outage (all of GitHub refusing) is not this op's fault, so it must
|
|
68
|
+
// not burn the op's backoff escalation (#208). The circuit breaker's
|
|
69
|
+
// fast-fail (`GhRateLimitError`) is the most common shape — its stderr
|
|
70
|
+
// says "circuit breaker open", which `isRateLimitRefusal`'s body regex
|
|
71
|
+
// does not match — so classify it by type first and honour its
|
|
72
|
+
// `retryAtMs`. Every other failure counts and doubles the next wait.
|
|
73
|
+
if (err instanceof GhRateLimitError) {
|
|
74
|
+
store.deferLabelOp(op.id, errText(err), err.retryAtMs, false);
|
|
75
|
+
} else {
|
|
76
|
+
const rateLimited = isRateLimitRefusal(err);
|
|
77
|
+
store.deferLabelOp(
|
|
78
|
+
op.id,
|
|
79
|
+
errText(err),
|
|
80
|
+
Date.now() + labelOpBackoffMs(op.attempts + 1),
|
|
81
|
+
!rateLimited,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
deferred += 1;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
// Every remaining op in this pass was deferred (parked behind its backoff),
|
|
88
|
+
// so re-querying would only re-select the same deferred rows; nothing more
|
|
89
|
+
// is applicable until a backoff elapses.
|
|
90
|
+
if (!progressed) break;
|
|
91
|
+
}
|
|
92
|
+
return { applied, deferred };
|
|
93
|
+
}
|
package/src/routing.ts
CHANGED
|
@@ -55,6 +55,26 @@ export function isEligible(issue: ReadyIssue, p: ProjectConfig): boolean {
|
|
|
55
55
|
return !labels.has(inProgress) && !labels.has(blocked) && !labels.has(failed);
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
+
/**
|
|
59
|
+
* The label set an issue *effectively* carries once the projection outbox is
|
|
60
|
+
* accounted for (#201). Pending ops apply in enqueue order — the same order
|
|
61
|
+
* the projector will apply them — so an issue with a pending queue-label
|
|
62
|
+
* removal drops out of eligibility immediately, and a pending state-label
|
|
63
|
+
* removal stops a stale GitHub label from blocking redispatch while the
|
|
64
|
+
* tracker is still catching up. Pure: the physical label set is untouched.
|
|
65
|
+
*/
|
|
66
|
+
export function effectiveLabels(
|
|
67
|
+
labels: readonly string[],
|
|
68
|
+
pending: readonly { op: "add" | "remove"; label: string }[],
|
|
69
|
+
): string[] {
|
|
70
|
+
const set = new Set(labels);
|
|
71
|
+
for (const op of pending) {
|
|
72
|
+
if (op.op === "add") set.add(op.label);
|
|
73
|
+
else set.delete(op.label);
|
|
74
|
+
}
|
|
75
|
+
return [...set];
|
|
76
|
+
}
|
|
77
|
+
|
|
58
78
|
/**
|
|
59
79
|
* Partition eligible issues into dispatchable and needs-a-human.
|
|
60
80
|
*
|
package/src/store.ts
CHANGED
|
@@ -26,6 +26,7 @@ import type {
|
|
|
26
26
|
FrictionKind,
|
|
27
27
|
FrictionObservation,
|
|
28
28
|
FrictionSignal,
|
|
29
|
+
LabelOp,
|
|
29
30
|
MergeLock,
|
|
30
31
|
ReportDeliveryState,
|
|
31
32
|
ReportDraft,
|
|
@@ -111,6 +112,7 @@ const UPDATABLE_COLUMNS: Record<string, true> = {
|
|
|
111
112
|
endedAt: true,
|
|
112
113
|
lastError: true,
|
|
113
114
|
settlementFlags: true,
|
|
115
|
+
report: true,
|
|
114
116
|
failureClass: true,
|
|
115
117
|
recoveryAction: true,
|
|
116
118
|
recoveredAt: true,
|
|
@@ -142,6 +144,7 @@ interface RunRow {
|
|
|
142
144
|
endedAt: number | null;
|
|
143
145
|
lastError: string | null;
|
|
144
146
|
settlementFlags: string | null;
|
|
147
|
+
report: string | null;
|
|
145
148
|
failureClass: string | null;
|
|
146
149
|
recoveryAction: string | null;
|
|
147
150
|
recoveredAt: number | null;
|
|
@@ -163,6 +166,35 @@ interface FrictionSurfaceRow {
|
|
|
163
166
|
at: number;
|
|
164
167
|
}
|
|
165
168
|
|
|
169
|
+
/** The `label_ops` table exactly as SQLite hands it back (#201). */
|
|
170
|
+
interface LabelOpRow {
|
|
171
|
+
id: number;
|
|
172
|
+
project: string;
|
|
173
|
+
issue: number;
|
|
174
|
+
op: "add" | "remove";
|
|
175
|
+
label: string;
|
|
176
|
+
createdAt: number;
|
|
177
|
+
attempts: number;
|
|
178
|
+
nextAttemptAt: number;
|
|
179
|
+
lastError: string | null;
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/** NULL columns become absent properties, matching the other row converters. */
|
|
183
|
+
function toLabelOp(row: LabelOpRow): LabelOp {
|
|
184
|
+
const op: LabelOp = {
|
|
185
|
+
id: row.id,
|
|
186
|
+
project: row.project,
|
|
187
|
+
issue: row.issue,
|
|
188
|
+
op: row.op,
|
|
189
|
+
label: row.label,
|
|
190
|
+
createdAt: row.createdAt,
|
|
191
|
+
attempts: row.attempts,
|
|
192
|
+
nextAttemptAt: row.nextAttemptAt,
|
|
193
|
+
};
|
|
194
|
+
if (row.lastError !== null) op.lastError = row.lastError;
|
|
195
|
+
return op;
|
|
196
|
+
}
|
|
197
|
+
|
|
166
198
|
/** The `reports` table exactly as SQLite hands it back. */
|
|
167
199
|
interface ReportRow {
|
|
168
200
|
id: string;
|
|
@@ -229,6 +261,7 @@ CREATE TABLE IF NOT EXISTS runs (
|
|
|
229
261
|
endedAt INTEGER,
|
|
230
262
|
lastError TEXT,
|
|
231
263
|
settlementFlags TEXT,
|
|
264
|
+
report TEXT,
|
|
232
265
|
failureClass TEXT,
|
|
233
266
|
recoveryAction TEXT,
|
|
234
267
|
recoveredAt INTEGER
|
|
@@ -360,6 +393,46 @@ CREATE TABLE IF NOT EXISTS decisions (
|
|
|
360
393
|
resolution TEXT
|
|
361
394
|
);
|
|
362
395
|
CREATE INDEX IF NOT EXISTS decisions_project_state ON decisions (project, state);
|
|
396
|
+
|
|
397
|
+
-- GitHub rate-limit refusals the tracker observed (#198). Written by the
|
|
398
|
+
-- daemon's tracker hook, not polled, so status can show what GitHub actually
|
|
399
|
+
-- refused next to the polled budget that might still look healthy. Only one
|
|
400
|
+
-- column: a refusal carries no id or dedupe key — rows are pruned by age
|
|
401
|
+
-- (24h) in the same write that inserts, kept long enough for status's 5m
|
|
402
|
+
-- window and for an operator who wants to see the last day of refusals.
|
|
403
|
+
CREATE TABLE IF NOT EXISTS gh_refusals (
|
|
404
|
+
at INTEGER NOT NULL
|
|
405
|
+
);
|
|
406
|
+
CREATE INDEX IF NOT EXISTS gh_refusals_at ON gh_refusals (at);
|
|
407
|
+
|
|
408
|
+
-- The daemon's tracked gh call count, per UTC day and call source (#198).
|
|
409
|
+
-- The single funnel in tracker/github.ts counts every spawn; the day/source
|
|
410
|
+
-- pair is the partition, so today's column and today's row survive a restart.
|
|
411
|
+
CREATE TABLE IF NOT EXISTS gh_calls (
|
|
412
|
+
day TEXT NOT NULL,
|
|
413
|
+
source TEXT NOT NULL,
|
|
414
|
+
calls INTEGER NOT NULL,
|
|
415
|
+
PRIMARY KEY (day, source)
|
|
416
|
+
);
|
|
417
|
+
-- The label projection outbox (#201). Every decided GitHub label change the
|
|
418
|
+
-- dispatcher made is a row here until the tracker has applied it; the daemon
|
|
419
|
+
-- retries with backoff, and eligibility reads pendingLabelOpsFor so a label
|
|
420
|
+
-- GitHub refuses to move cannot strand the issue the daemon already decided
|
|
421
|
+
-- to move on. The queue-label add and the state-label remove of one swap are
|
|
422
|
+
-- two rows, applied strictly in id order, which is what makes a swap atomic:
|
|
423
|
+
-- the remove cannot land before the add it was enqueued after.
|
|
424
|
+
CREATE TABLE IF NOT EXISTS label_ops (
|
|
425
|
+
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
426
|
+
project TEXT NOT NULL,
|
|
427
|
+
issue INTEGER NOT NULL,
|
|
428
|
+
op TEXT NOT NULL CHECK (op IN ('add', 'remove')),
|
|
429
|
+
label TEXT NOT NULL,
|
|
430
|
+
createdAt INTEGER NOT NULL,
|
|
431
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
432
|
+
nextAttemptAt INTEGER NOT NULL DEFAULT 0,
|
|
433
|
+
lastError TEXT
|
|
434
|
+
);
|
|
435
|
+
CREATE INDEX IF NOT EXISTS label_ops_project_next ON label_ops (project, nextAttemptAt);
|
|
363
436
|
`;
|
|
364
437
|
|
|
365
438
|
/**
|
|
@@ -435,6 +508,7 @@ function toRecord(row: RunRow): RunRecord {
|
|
|
435
508
|
const flags = toSettlementFlags(row.settlementFlags);
|
|
436
509
|
if (flags !== undefined) record.settlementFlags = flags;
|
|
437
510
|
}
|
|
511
|
+
if (row.report !== null) record.report = row.report;
|
|
438
512
|
if (row.failureClass !== null) record.failureClass = row.failureClass as FailureClass;
|
|
439
513
|
if (row.recoveryAction !== null) record.recoveryAction = row.recoveryAction as RecoveryAction;
|
|
440
514
|
if (row.recoveredAt !== null) record.recoveredAt = row.recoveredAt;
|
|
@@ -613,6 +687,12 @@ export function dbPath(): string {
|
|
|
613
687
|
return join(stateDir(), "conductor.db");
|
|
614
688
|
}
|
|
615
689
|
|
|
690
|
+
/** The UTC calendar day (`YYYY-MM-DD`) a moment falls on — the partition key
|
|
691
|
+
* for the daemon's observed github call counter (#198). */
|
|
692
|
+
export function utcDay(now: number = Date.now()): string {
|
|
693
|
+
return new Date(now).toISOString().slice(0, 10);
|
|
694
|
+
}
|
|
695
|
+
|
|
616
696
|
/**
|
|
617
697
|
* Open (creating if needed) the run store at `dbPath`; `:memory:` is honoured
|
|
618
698
|
* for tests. Safe to call on a fresh path — the schema is applied on open, so
|
|
@@ -665,6 +745,13 @@ export function openStore(dbPath: string): Store {
|
|
|
665
745
|
if (!columns.some((column) => column.name === "settlementFlags")) {
|
|
666
746
|
db.exec("ALTER TABLE runs ADD COLUMN settlementFlags TEXT");
|
|
667
747
|
}
|
|
748
|
+
// Rows written before the settlement report was persisted (#199) have no
|
|
749
|
+
// text to pool across attempts, so a continuation reads empty priors and
|
|
750
|
+
// audits exactly as it did before this release. No backfill: the bytes only
|
|
751
|
+
// ever existed in a worker's memory.
|
|
752
|
+
if (!columns.some((column) => column.name === "report")) {
|
|
753
|
+
db.exec("ALTER TABLE runs ADD COLUMN report TEXT");
|
|
754
|
+
}
|
|
668
755
|
// Every row written before #132 is unclassified, and NULL is the honest
|
|
669
756
|
// reading of that: the budget counters below deliberately still count an
|
|
670
757
|
// unclassified terminal row exactly as this release's predecessor did, so an
|
|
@@ -772,8 +859,8 @@ export function openStore(dbPath: string): Store {
|
|
|
772
859
|
`INSERT INTO runs (
|
|
773
860
|
id, project, issue, repo, branch, worktree, state, attempt, turns,
|
|
774
861
|
maxTurns, spendUsd, sessionFile, prUrl, headSha, salvageSha, salvageError,
|
|
775
|
-
salvageAckAt, startedAt, endedAt, lastError, settlementFlags
|
|
776
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
862
|
+
salvageAckAt, startedAt, endedAt, lastError, settlementFlags, report
|
|
863
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
777
864
|
);
|
|
778
865
|
const selectRun = db.query<RunRow, [string]>(`SELECT * FROM runs WHERE id = ?`);
|
|
779
866
|
const selectActive = db.query<RunRow, SqlValue[]>(
|
|
@@ -886,6 +973,14 @@ export function openStore(dbPath: string): Store {
|
|
|
886
973
|
ORDER BY startedAt DESC, rowid DESC
|
|
887
974
|
LIMIT 1`,
|
|
888
975
|
);
|
|
976
|
+
// Every attempt that settled with a report, in attempt order. NULLs are the
|
|
977
|
+
// rows written before #199, or one killed before the worker returned a
|
|
978
|
+
// report — skipping them makes a continuation read exactly today's priors.
|
|
979
|
+
const selectAttemptReports = db.query<{ attempt: number; report: string }, [string, number]>(
|
|
980
|
+
`SELECT attempt, report FROM runs
|
|
981
|
+
WHERE project = ? AND issue = ? AND report IS NOT NULL
|
|
982
|
+
ORDER BY attempt`,
|
|
983
|
+
);
|
|
889
984
|
const countStartedSince = db.query<{ n: number }, [string, number]>(
|
|
890
985
|
`SELECT COUNT(*) AS n FROM runs WHERE project = ? AND startedAt >= ?`,
|
|
891
986
|
);
|
|
@@ -938,6 +1033,31 @@ export function openStore(dbPath: string): Store {
|
|
|
938
1033
|
ON CONFLICT(project, kind) DO UPDATE SET at = excluded.at`,
|
|
939
1034
|
);
|
|
940
1035
|
|
|
1036
|
+
// Observed GitHub rate-limit refusals (#198). Insert + prune share one
|
|
1037
|
+
// transaction so a burst never grows the table without bound.
|
|
1038
|
+
const insertGhRefusal = db.query<unknown, [number]>(
|
|
1039
|
+
`INSERT INTO gh_refusals (at) VALUES (?)`,
|
|
1040
|
+
);
|
|
1041
|
+
const pruneGhRefusals = db.query<unknown, [number]>(
|
|
1042
|
+
`DELETE FROM gh_refusals WHERE at < ?`,
|
|
1043
|
+
);
|
|
1044
|
+
const selectGhRefusalsSince = db.query<{ n: number; latest: number | null }, [number]>(
|
|
1045
|
+
`SELECT COUNT(*) AS n, MAX(at) AS latest FROM gh_refusals WHERE at >= ?`,
|
|
1046
|
+
);
|
|
1047
|
+
const recordGhRefusalTx = db.transaction((at: number): void => {
|
|
1048
|
+
insertGhRefusal.run(at);
|
|
1049
|
+
pruneGhRefusals.run(at - 24 * 60 * 60 * 1000);
|
|
1050
|
+
});
|
|
1051
|
+
|
|
1052
|
+
// The daemon's tracked github call counts (#198): one UPSERT per spawn.
|
|
1053
|
+
const bumpGhCall = db.query<unknown, [string, string]>(
|
|
1054
|
+
`INSERT INTO gh_calls (day, source, calls) VALUES (?, ?, 1)
|
|
1055
|
+
ON CONFLICT(day, source) DO UPDATE SET calls = calls + 1`,
|
|
1056
|
+
);
|
|
1057
|
+
const selectGhCalls = db.query<{ source: string; calls: number }, [string]>(
|
|
1058
|
+
`SELECT source, calls FROM gh_calls WHERE day = ? ORDER BY source`,
|
|
1059
|
+
);
|
|
1060
|
+
|
|
941
1061
|
// The report outbox (#123). Every terminal transition names the attempt it is
|
|
942
1062
|
// settling, because a request can come back *after* the stale-`sending` sweep
|
|
943
1063
|
// has already reclaimed its row — and a late answer must not overwrite a
|
|
@@ -1071,6 +1191,72 @@ export function openStore(dbPath: string): Store {
|
|
|
1071
1191
|
},
|
|
1072
1192
|
);
|
|
1073
1193
|
|
|
1194
|
+
// The label projection outbox (#201). A decided label change is a row until
|
|
1195
|
+
// the tracker has applied it; the projector drains in id order, and one
|
|
1196
|
+
// issue's ops are atomic as a run — a failed op parks the rest of its issue
|
|
1197
|
+
// for the next pass rather than letting a later op land out of order.
|
|
1198
|
+
const insertLabelOp = db.query<unknown, SqlValue[]>(
|
|
1199
|
+
`INSERT INTO label_ops (project, issue, op, label, createdAt) VALUES (?, ?, ?, ?, ?)`,
|
|
1200
|
+
);
|
|
1201
|
+
const selectPendingLabelOps = db.query<LabelOpRow, [string, number]>(
|
|
1202
|
+
`SELECT o.* FROM label_ops AS o
|
|
1203
|
+
WHERE o.project = ? AND o.nextAttemptAt <= ?
|
|
1204
|
+
AND NOT EXISTS (
|
|
1205
|
+
SELECT 1 FROM label_ops AS e
|
|
1206
|
+
WHERE e.project = o.project AND e.issue = o.issue AND e.id < o.id
|
|
1207
|
+
)
|
|
1208
|
+
ORDER BY o.id`,
|
|
1209
|
+
);
|
|
1210
|
+
const selectPendingLabelOpsFor = db.query<LabelOpRow, [string, number]>(
|
|
1211
|
+
`SELECT * FROM label_ops WHERE project = ? AND issue = ? ORDER BY id`,
|
|
1212
|
+
);
|
|
1213
|
+
const selectOldestPendingLabelOp = db.query<LabelOpRow, [string]>(
|
|
1214
|
+
`SELECT * FROM label_ops WHERE project = ? ORDER BY createdAt ASC, id ASC LIMIT 1`,
|
|
1215
|
+
);
|
|
1216
|
+
const countLabelOps = db.query<{ n: number }, [string]>(
|
|
1217
|
+
`SELECT COUNT(*) AS n FROM label_ops WHERE project = ?`,
|
|
1218
|
+
);
|
|
1219
|
+
const deleteLabelOp = db.query<unknown, [number]>(`DELETE FROM label_ops WHERE id = ?`);
|
|
1220
|
+
const bumpLabelOpAttempt = db.query<unknown, [string, number, number]>(
|
|
1221
|
+
`UPDATE label_ops SET attempts = attempts + 1, lastError = ?, nextAttemptAt = ? WHERE id = ?`,
|
|
1222
|
+
);
|
|
1223
|
+
// Rate-limit refusals defer WITHOUT counting an attempt: a shared outage is
|
|
1224
|
+
// not the op's fault, so it must not burn the backoff escalation (#208).
|
|
1225
|
+
const parkLabelOp = db.query<unknown, [string, number, number]>(
|
|
1226
|
+
`UPDATE label_ops SET lastError = ?, nextAttemptAt = ? WHERE id = ?`,
|
|
1227
|
+
);
|
|
1228
|
+
// Coalesce: if the LATEST pending op for the same project+issue+label is
|
|
1229
|
+
// already the same operation, adding another is a duplicate that would sit
|
|
1230
|
+
// parked behind the oldest pending one, growing unboundedly (a reconcile
|
|
1231
|
+
// re-offering a removal the tracker keeps refusing). Only the latest matters:
|
|
1232
|
+
// a `remove → add → remove` sequence must keep the final remove even though
|
|
1233
|
+
// the first remove is still owed, because the add in between is a real
|
|
1234
|
+
// transition the final remove answers. Opposite transitions are never
|
|
1235
|
+
// coalesced. The check and insert share the transaction so they cannot race
|
|
1236
|
+
// (#201).
|
|
1237
|
+
const selectLatestLabelOp = db.query<{ op: "add" | "remove" }, [string, number, string]>(
|
|
1238
|
+
`SELECT op FROM label_ops
|
|
1239
|
+
WHERE project = ? AND issue = ? AND label = ?
|
|
1240
|
+
ORDER BY id DESC LIMIT 1`,
|
|
1241
|
+
);
|
|
1242
|
+
const enqueueLabelOpsTx = db.transaction(
|
|
1243
|
+
(
|
|
1244
|
+
rows: readonly {
|
|
1245
|
+
project: string;
|
|
1246
|
+
issue: number;
|
|
1247
|
+
op: "add" | "remove";
|
|
1248
|
+
label: string;
|
|
1249
|
+
createdAt: number;
|
|
1250
|
+
}[],
|
|
1251
|
+
): void => {
|
|
1252
|
+
for (const row of rows) {
|
|
1253
|
+
const latest = selectLatestLabelOp.get(row.project, row.issue, row.label);
|
|
1254
|
+
if (latest !== undefined && latest !== null && latest.op === row.op) continue;
|
|
1255
|
+
insertLabelOp.run(toSql(row.project), row.issue, row.op, row.label, row.createdAt);
|
|
1256
|
+
}
|
|
1257
|
+
},
|
|
1258
|
+
);
|
|
1259
|
+
|
|
1074
1260
|
const recordFriction = (project: string, observation: FrictionObservation): void => {
|
|
1075
1261
|
if (
|
|
1076
1262
|
!Number.isSafeInteger(observation.occurrences) ||
|
|
@@ -1108,6 +1294,12 @@ export function openStore(dbPath: string): Store {
|
|
|
1108
1294
|
);
|
|
1109
1295
|
};
|
|
1110
1296
|
|
|
1297
|
+
const recordGhRefusal = (at: number): void => {
|
|
1298
|
+
// Same guard as `recordFriction`: a bad clock must not corrupt the store.
|
|
1299
|
+
if (!Number.isSafeInteger(at) || at < 0) return;
|
|
1300
|
+
recordGhRefusalTx(at);
|
|
1301
|
+
};
|
|
1302
|
+
|
|
1111
1303
|
return {
|
|
1112
1304
|
createRun(r: Omit<RunRecord, "id">): RunRecord {
|
|
1113
1305
|
const record: RunRecord = { ...r, id: crypto.randomUUID() };
|
|
@@ -1133,6 +1325,7 @@ export function openStore(dbPath: string): Store {
|
|
|
1133
1325
|
toSql(record.endedAt),
|
|
1134
1326
|
toSql(record.lastError),
|
|
1135
1327
|
toSql(record.settlementFlags),
|
|
1328
|
+
toSql(record.report),
|
|
1136
1329
|
);
|
|
1137
1330
|
return record;
|
|
1138
1331
|
},
|
|
@@ -1200,6 +1393,10 @@ export function openStore(dbPath: string): Store {
|
|
|
1200
1393
|
return row ? toRecord(row) : undefined;
|
|
1201
1394
|
},
|
|
1202
1395
|
|
|
1396
|
+
attemptReports(project: string, issue: number): { attempt: number; report: string }[] {
|
|
1397
|
+
return selectAttemptReports.all(project, issue);
|
|
1398
|
+
},
|
|
1399
|
+
|
|
1203
1400
|
|
|
1204
1401
|
runsStartedSince(project: string, sinceEpochMs: number): number {
|
|
1205
1402
|
return countStartedSince.get(project, sinceEpochMs)?.n ?? 0;
|
|
@@ -1250,6 +1447,24 @@ export function openStore(dbPath: string): Store {
|
|
|
1250
1447
|
|
|
1251
1448
|
recordFriction,
|
|
1252
1449
|
|
|
1450
|
+
recordGhRefusal,
|
|
1451
|
+
|
|
1452
|
+
ghRefusalsSince(sinceMs: number): { count: number; latestAt?: number } {
|
|
1453
|
+
const row = selectGhRefusalsSince.get(sinceMs);
|
|
1454
|
+
if (row === null || row === undefined || row.n === 0 || row.latest === null) {
|
|
1455
|
+
return { count: 0 };
|
|
1456
|
+
}
|
|
1457
|
+
return { count: row.n, latestAt: row.latest };
|
|
1458
|
+
},
|
|
1459
|
+
|
|
1460
|
+
bumpGhCalls(day: string, source: string): void {
|
|
1461
|
+
bumpGhCall.run(day, source);
|
|
1462
|
+
},
|
|
1463
|
+
|
|
1464
|
+
ghCallsToday(day: string): { source: string; calls: number }[] {
|
|
1465
|
+
return selectGhCalls.all(day);
|
|
1466
|
+
},
|
|
1467
|
+
|
|
1253
1468
|
pendingFriction(
|
|
1254
1469
|
project: string,
|
|
1255
1470
|
sinceEpochMs: number,
|
|
@@ -1532,6 +1747,43 @@ export function openStore(dbPath: string): Store {
|
|
|
1532
1747
|
return row === null ? undefined : { ...row };
|
|
1533
1748
|
},
|
|
1534
1749
|
|
|
1750
|
+
enqueueLabelOps(
|
|
1751
|
+
project: string,
|
|
1752
|
+
ops: readonly { issue: number; op: "add" | "remove"; label: string }[],
|
|
1753
|
+
): void {
|
|
1754
|
+
if (ops.length === 0) return;
|
|
1755
|
+
const now = Date.now();
|
|
1756
|
+
enqueueLabelOpsTx(
|
|
1757
|
+
ops.map((op) => ({ ...op, project, createdAt: now })),
|
|
1758
|
+
);
|
|
1759
|
+
},
|
|
1760
|
+
|
|
1761
|
+
pendingLabelOps(project: string, now: number): LabelOp[] {
|
|
1762
|
+
return selectPendingLabelOps.all(project, now).map(toLabelOp);
|
|
1763
|
+
},
|
|
1764
|
+
|
|
1765
|
+
pendingLabelOpsFor(project: string, issue: number): LabelOp[] {
|
|
1766
|
+
return selectPendingLabelOpsFor.all(project, issue).map(toLabelOp);
|
|
1767
|
+
},
|
|
1768
|
+
|
|
1769
|
+
settleLabelOp(id: number): void {
|
|
1770
|
+
deleteLabelOp.run(id);
|
|
1771
|
+
},
|
|
1772
|
+
|
|
1773
|
+
deferLabelOp(id: number, error: string, nextAttemptAt: number, countAttempt = true): void {
|
|
1774
|
+
if (countAttempt) bumpLabelOpAttempt.run(error, nextAttemptAt, id);
|
|
1775
|
+
else parkLabelOp.run(error, nextAttemptAt, id);
|
|
1776
|
+
},
|
|
1777
|
+
|
|
1778
|
+
countPendingLabelOps(project: string): number {
|
|
1779
|
+
return countLabelOps.get(project)?.n ?? 0;
|
|
1780
|
+
},
|
|
1781
|
+
|
|
1782
|
+
oldestPendingLabelOpAt(project: string): number | undefined {
|
|
1783
|
+
const row = selectOldestPendingLabelOp.get(project);
|
|
1784
|
+
return row === null ? undefined : row.createdAt;
|
|
1785
|
+
},
|
|
1786
|
+
|
|
1535
1787
|
close(): void {
|
|
1536
1788
|
db.close(false);
|
|
1537
1789
|
},
|