omp-conductor 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +45 -21
- package/package.json +1 -1
- package/src/backups.ts +46 -0
- package/src/board.ts +254 -38
- package/src/brief-upgrade.ts +1 -30
- package/src/cli.ts +17 -5
- package/src/config.ts +13 -0
- package/src/daemon.ts +238 -123
- package/src/diff-flags.ts +35 -6
- package/src/fleet.ts +28 -2
- package/src/label-projection.ts +93 -0
- package/src/routing.ts +20 -0
- package/src/store.ts +254 -2
- package/src/tracker/github.ts +266 -18
- package/src/types.ts +75 -3
- package/src/unblock.ts +70 -19
- package/src/upgrade.ts +28 -4
package/src/daemon.ts
CHANGED
|
@@ -32,11 +32,12 @@ import { startOrchestrator } from "./orchestrator.ts";
|
|
|
32
32
|
import type { OrchestratorHandle } from "./orchestrator.ts";
|
|
33
33
|
import { createReportOutbox, formatOpenReports } from "./reports.ts";
|
|
34
34
|
import { recordReleaseBlock } from "./release-policy.ts";
|
|
35
|
-
import { branchName, route } from "./routing.ts";
|
|
35
|
+
import { branchName, effectiveLabels, route } from "./routing.ts";
|
|
36
36
|
import type { Routed, UnroutableReason } from "./routing.ts";
|
|
37
37
|
import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
|
|
38
38
|
import { classifyRun, type ClassifyFacts } from "./failure-class.ts";
|
|
39
|
-
import {
|
|
39
|
+
import { projectLabels } from "./label-projection.ts";
|
|
40
|
+
import { dbPath, openStore, utcDay } from "./store.ts";
|
|
40
41
|
import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
|
|
41
42
|
import { RELEASE_SHAPES } from "./types.ts";
|
|
42
43
|
import type {
|
|
@@ -515,13 +516,17 @@ function acceptanceCriteria(issue: ReadyIssue): string {
|
|
|
515
516
|
}
|
|
516
517
|
|
|
517
518
|
/**
|
|
518
|
-
*
|
|
519
|
-
*
|
|
520
|
-
* shape `isEligible` treats as fresh
|
|
519
|
+
* Record a state-label swap for projection (#201). The add enqueues before the
|
|
520
|
+
* remove — the reverse order would leave a window where the issue carries no
|
|
521
|
+
* state label at all, which is exactly the shape `isEligible` treats as fresh
|
|
522
|
+
* work. Synchronous: enqueueing is a local store write and cannot fail on the
|
|
523
|
+
* tracker; the projector applies the pair in order and retries on refusal.
|
|
521
524
|
*/
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
+
function swapLabel(store: Store, projectName: string, issue: number, from: string, to: string): void {
|
|
526
|
+
store.enqueueLabelOps(projectName, [
|
|
527
|
+
{ issue, op: "add", label: to },
|
|
528
|
+
{ issue, op: "remove", label: from },
|
|
529
|
+
]);
|
|
525
530
|
}
|
|
526
531
|
|
|
527
532
|
/**
|
|
@@ -855,13 +860,18 @@ export async function verifyPushedGreenClaim(
|
|
|
855
860
|
*/
|
|
856
861
|
export async function collectSettlementFlags(
|
|
857
862
|
tracker: Pick<Tracker, "prDiff">,
|
|
858
|
-
claim: { prUrl?: string; report: string; issueText: string },
|
|
863
|
+
claim: { prUrl?: string; report: string; priorReports?: readonly string[]; issueText: string },
|
|
859
864
|
): Promise<{ flags: SettlementFlag[]; truncated: boolean } | undefined> {
|
|
860
865
|
if (claim.prUrl === undefined) return undefined;
|
|
861
866
|
const diff = await tracker.prDiff(claim.prUrl);
|
|
862
867
|
if (diff === undefined) return undefined;
|
|
863
868
|
return {
|
|
864
|
-
flags: analyseSettlement({
|
|
869
|
+
flags: analyseSettlement({
|
|
870
|
+
report: claim.report,
|
|
871
|
+
issueText: claim.issueText,
|
|
872
|
+
diff,
|
|
873
|
+
priorReports: claim.priorReports,
|
|
874
|
+
}),
|
|
865
875
|
truncated: diff.truncated,
|
|
866
876
|
};
|
|
867
877
|
}
|
|
@@ -938,12 +948,17 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
938
948
|
};
|
|
939
949
|
|
|
940
950
|
try {
|
|
941
|
-
// Claim on the
|
|
942
|
-
//
|
|
943
|
-
//
|
|
944
|
-
//
|
|
945
|
-
|
|
946
|
-
|
|
951
|
+
// Claim on the STORE first, before anything that can fail. The run row —
|
|
952
|
+
// not the label — is the crash-safe guard against double dispatch: rows
|
|
953
|
+
// are local, written before any network call, and the startup orphan
|
|
954
|
+
// sweep marks process-less claimed/running rows `orphaned`, so a daemon
|
|
955
|
+
// dying mid-claim leaves a row a human can triage instead of a label only
|
|
956
|
+
// the orchestrator may touch. The in-progress label is a write-behind
|
|
957
|
+
// projection of that row: enqueued here, flushed by the post-admission
|
|
958
|
+
// projection pass in the same tick on the healthy path, and — while
|
|
959
|
+
// pending — kept effective off the ready set by the routing overlay, so a
|
|
960
|
+
// crash between the row and its label cannot double-dispatch either
|
|
961
|
+
// (#201).
|
|
947
962
|
// Read before this attempt's own row exists, so `latestRun` still means the
|
|
948
963
|
// attempt whose work this one inherits.
|
|
949
964
|
const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
|
|
@@ -963,6 +978,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
963
978
|
startedAt: Date.now(),
|
|
964
979
|
});
|
|
965
980
|
const runId = run.id;
|
|
981
|
+
store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
|
|
982
|
+
claimed = true;
|
|
966
983
|
turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
|
|
967
984
|
|
|
968
985
|
// A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
|
|
@@ -1102,6 +1119,15 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1102
1119
|
? await collectSettlementFlags(tracker, {
|
|
1103
1120
|
prUrl: result.prUrl,
|
|
1104
1121
|
report: result.report,
|
|
1122
|
+
// Earlier attempts' reports pool as disclosure coverage (#199). The
|
|
1123
|
+
// current row has not settled yet, so this is exactly the prior set;
|
|
1124
|
+
// the strict `< attempt` guard keeps an already-persisted copy of
|
|
1125
|
+
// this attempt out just in case. Pre-#199 rows have NULL reports and
|
|
1126
|
+
// contribute nothing — today's behaviour.
|
|
1127
|
+
priorReports: store
|
|
1128
|
+
.attemptReports(project.name, issue)
|
|
1129
|
+
.filter((r) => r.attempt < attempt)
|
|
1130
|
+
.map((r) => r.report),
|
|
1105
1131
|
issueText: `${r.issue.title}\n${r.issue.body}`,
|
|
1106
1132
|
})
|
|
1107
1133
|
: undefined;
|
|
@@ -1110,7 +1136,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1110
1136
|
else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
|
|
1111
1137
|
}
|
|
1112
1138
|
const auditLines =
|
|
1113
|
-
audit === undefined
|
|
1139
|
+
audit === undefined
|
|
1140
|
+
? []
|
|
1141
|
+
: formatSettlementFlags(audit.flags, { truncated: audit.truncated, attempts: attempt });
|
|
1114
1142
|
|
|
1115
1143
|
const finalReport = [
|
|
1116
1144
|
...(verified.reason === undefined ? [] : [verified.reason, ""]),
|
|
@@ -1160,6 +1188,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1160
1188
|
prUrl: result.prUrl,
|
|
1161
1189
|
headSha: result.headSha,
|
|
1162
1190
|
sessionFile: result.sessionFile,
|
|
1191
|
+
// Every terminal state persists the worker's report, not just a green
|
|
1192
|
+
// push: the report of a killed attempt is exactly the one a later
|
|
1193
|
+
// continuation must pool its disclosures from (#199).
|
|
1194
|
+
report: result.report,
|
|
1163
1195
|
...(verified.reason === undefined ? {} : { lastError: verified.reason }),
|
|
1164
1196
|
...settlement?.patch,
|
|
1165
1197
|
...(audit === undefined || audit.flags.length === 0
|
|
@@ -1170,7 +1202,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1170
1202
|
const salvaged = settlement?.lines ?? [];
|
|
1171
1203
|
|
|
1172
1204
|
if (state === "blocked") {
|
|
1173
|
-
|
|
1205
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
|
|
1174
1206
|
await safeEscalate(d, {
|
|
1175
1207
|
tier: 1,
|
|
1176
1208
|
project: project.name,
|
|
@@ -1193,8 +1225,12 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1193
1225
|
});
|
|
1194
1226
|
|
|
1195
1227
|
if (continueTurns) {
|
|
1196
|
-
|
|
1197
|
-
|
|
1228
|
+
// Requeue as one ordered pair: the in-progress removal before the
|
|
1229
|
+
// queue add, exactly the order the projector will apply them in (#201).
|
|
1230
|
+
store.enqueueLabelOps(project.name, [
|
|
1231
|
+
{ issue, op: "remove", label: inProgress },
|
|
1232
|
+
{ issue, op: "add", label: project.queueLabel },
|
|
1233
|
+
]);
|
|
1198
1234
|
log(
|
|
1199
1235
|
`#${issue} turns-cap on run ${attempt}, continuation ` +
|
|
1200
1236
|
`${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
|
|
@@ -1218,7 +1254,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1218
1254
|
].join("\n"),
|
|
1219
1255
|
});
|
|
1220
1256
|
} else {
|
|
1221
|
-
|
|
1257
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1222
1258
|
await safeEscalate(d, {
|
|
1223
1259
|
tier: 1,
|
|
1224
1260
|
project: project.name,
|
|
@@ -1316,11 +1352,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
|
|
|
1316
1352
|
if (claimed) {
|
|
1317
1353
|
// Leaving the issue stuck as in-progress would hide it from both the
|
|
1318
1354
|
// queue and the human, so relabel even on the error path.
|
|
1319
|
-
|
|
1320
|
-
|
|
1321
|
-
|
|
1322
|
-
log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
|
|
1323
|
-
}
|
|
1355
|
+
// The failure-path relabel cannot throw: it is a local outbox write, and
|
|
1356
|
+
// the projector retries until the tracker takes it (#201).
|
|
1357
|
+
swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
|
|
1324
1358
|
}
|
|
1325
1359
|
|
|
1326
1360
|
const salvaged = settlement?.lines ?? [];
|
|
@@ -1394,46 +1428,35 @@ export function settlementFor(pr: PrState | undefined, prUrl: string): Settlemen
|
|
|
1394
1428
|
}
|
|
1395
1429
|
|
|
1396
1430
|
/**
|
|
1397
|
-
*
|
|
1431
|
+
* Records the in-progress label's release as a projection op (#201).
|
|
1398
1432
|
*
|
|
1399
1433
|
* Settlement used to write only half of what it knew. On 2026-08-09 that cost
|
|
1400
1434
|
* the reference fleet two issues in one night: veltro#331 settled to `failed`
|
|
1401
1435
|
* at 23:53Z once the orchestrator closed veltro#332 unmerged, and veltro#344
|
|
1402
1436
|
* settled to `merged` at 05:54Z once chad#452 squash-merged. Both rows left the
|
|
1403
1437
|
* active set correctly; an authoritative `gh issue view` on each afterwards
|
|
1404
|
-
* still showed `agent:in-progress
|
|
1405
|
-
*
|
|
1406
|
-
* hand-editing one, and `unblock` refused to clear that particular label — so
|
|
1407
|
-
* both issues were permanently unclaimable with no supported way back (#18).
|
|
1438
|
+
* still showed `agent:in-progress` — permanently unclaimable with no supported
|
|
1439
|
+
* way back (#18).
|
|
1408
1440
|
*
|
|
1409
|
-
*
|
|
1410
|
-
*
|
|
1411
|
-
*
|
|
1412
|
-
*
|
|
1413
|
-
*
|
|
1414
|
-
*
|
|
1415
|
-
*
|
|
1416
|
-
*
|
|
1417
|
-
* will find it.
|
|
1441
|
+
* The outbox makes the row transition and the label one fact again: the
|
|
1442
|
+
* removal is enqueued in the same breath as the row is terminalised, the
|
|
1443
|
+
* projector applies it with unbounded retry, and while it is pending the
|
|
1444
|
+
* eligibility overlay treats the label as already gone. A tracker that refuses
|
|
1445
|
+
* the write (403, rate limit) can no longer strand the row — that is `#184`
|
|
1446
|
+
* and `#198` closed. No `pushed-*` row is ever written terminal with its
|
|
1447
|
+
* label release owed but unrecorded, because enqueueing is a local store write
|
|
1448
|
+
* that cannot fail on the tracker.
|
|
1418
1449
|
*
|
|
1419
|
-
*
|
|
1420
|
-
* adapter treats an absent label as a no-op, so false means the tracker could
|
|
1421
|
-
* not be reached or refused — a condition that passes.
|
|
1450
|
+
* Synchronous. The op is durable the moment this returns.
|
|
1422
1451
|
*/
|
|
1423
|
-
export
|
|
1424
|
-
d: Pick<Deps, "project" | "
|
|
1452
|
+
export function releaseInProgress(
|
|
1453
|
+
d: Pick<Deps, "project" | "store">,
|
|
1425
1454
|
issue: number,
|
|
1426
1455
|
why: string,
|
|
1427
|
-
):
|
|
1456
|
+
): void {
|
|
1428
1457
|
const label = d.project.stateLabels.inProgress;
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
log(`#${issue} released ${label}: ${why}`);
|
|
1432
|
-
return true;
|
|
1433
|
-
} catch (err) {
|
|
1434
|
-
log(`#${issue} could not release ${label} (${errText(err)}) — ${why}; retrying next tick`);
|
|
1435
|
-
return false;
|
|
1436
|
-
}
|
|
1458
|
+
d.store.enqueueLabelOps(d.project.name, [{ issue, op: "remove", label }]);
|
|
1459
|
+
log(`#${issue} released ${label} (queued): ${why}`);
|
|
1437
1460
|
}
|
|
1438
1461
|
|
|
1439
1462
|
/**
|
|
@@ -1507,15 +1530,15 @@ export async function settlePushedGreen(
|
|
|
1507
1530
|
|
|
1508
1531
|
const settlement = settlementFor(pr, run.prUrl);
|
|
1509
1532
|
if (settlement !== undefined) {
|
|
1510
|
-
//
|
|
1511
|
-
//
|
|
1512
|
-
//
|
|
1513
|
-
//
|
|
1514
|
-
//
|
|
1515
|
-
//
|
|
1516
|
-
//
|
|
1517
|
-
//
|
|
1518
|
-
|
|
1533
|
+
// The label removal and the terminal row are one fact again (#201): the
|
|
1534
|
+
// release is enqueued — a durable local write that cannot fail on the
|
|
1535
|
+
// tracker — in the same breath as the row is terminalised, so there is
|
|
1536
|
+
// no window in which a row beyond every later tick still owes its label.
|
|
1537
|
+
// The projector applies it with retry; while pending, the eligibility
|
|
1538
|
+
// overlay treats the label as already gone, so #18's
|
|
1539
|
+
// permanent-`agent:in-progress` cannot re-form even when GitHub refuses
|
|
1540
|
+
// the write.
|
|
1541
|
+
releaseInProgress(d, run.issue, settlement.reason);
|
|
1519
1542
|
const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
|
|
1520
1543
|
if (settlement.state === "failed") patch.lastError = settlement.reason;
|
|
1521
1544
|
store.updateRun(run.id, patch);
|
|
@@ -1536,11 +1559,11 @@ export async function settlePushedGreen(
|
|
|
1536
1559
|
store.updateRun(run.id, { state: "pushed-green", lastError: undefined });
|
|
1537
1560
|
log(`#${run.issue} checks settled: ${verification.reason}`);
|
|
1538
1561
|
} else if (verification.status === "failed") {
|
|
1539
|
-
// Equally terminal, so the
|
|
1540
|
-
//
|
|
1541
|
-
// releases nothing — that row is still awaiting a merge,
|
|
1542
|
-
// is exactly the work the label must keep guarding.
|
|
1543
|
-
|
|
1562
|
+
// Equally terminal, so the release is enqueued before the row writes,
|
|
1563
|
+
// for the same reason as the settlement branch above (see there). The
|
|
1564
|
+
// green branch releases nothing — that row is still awaiting a merge,
|
|
1565
|
+
// and its live PR is exactly the work the label must keep guarding.
|
|
1566
|
+
releaseInProgress(d, run.issue, verification.reason);
|
|
1544
1567
|
store.updateRun(run.id, { state: "failed", lastError: verification.reason });
|
|
1545
1568
|
log(`#${run.issue} checks failed: ${verification.reason}`);
|
|
1546
1569
|
} else {
|
|
@@ -1822,10 +1845,12 @@ export async function admitCandidates(
|
|
|
1822
1845
|
return { admitted: [], holds };
|
|
1823
1846
|
}
|
|
1824
1847
|
|
|
1825
|
-
// parent -> blocking issue. Seeded from active runs (including
|
|
1826
|
-
// then extended by candidates admitted earlier in this same
|
|
1827
|
-
// siblings never both clear the gate in one tick.
|
|
1828
|
-
|
|
1848
|
+
// parent -> repo name -> blocking issue. Seeded from active runs (including
|
|
1849
|
+
// pushed-green), then extended by candidates admitted earlier in this same
|
|
1850
|
+
// pass so two siblings of one epic never both clear the gate in one tick.
|
|
1851
|
+
// A busy issue whose run row cannot be resolved occupies the sentinel repo
|
|
1852
|
+
// "" — treated as matching every repo, failing toward holding (#197).
|
|
1853
|
+
const occupiedParents = new Map<number, Map<string, number>>();
|
|
1829
1854
|
const parentCache = new Map<number, number | undefined>();
|
|
1830
1855
|
|
|
1831
1856
|
const resolveParent = async (issue: number): Promise<number | undefined> => {
|
|
@@ -1840,8 +1865,18 @@ export async function admitCandidates(
|
|
|
1840
1865
|
for (const issue of busyIssues) {
|
|
1841
1866
|
try {
|
|
1842
1867
|
const parent = await resolveParent(issue);
|
|
1843
|
-
if (parent
|
|
1844
|
-
|
|
1868
|
+
if (parent === undefined) continue;
|
|
1869
|
+
// The runs table records which repo each attempt worked in, and sibling
|
|
1870
|
+
// holds are now per-repo, so a busy child only occupies its epic under
|
|
1871
|
+
// that repo's name (same spelling as `createRun` writes from
|
|
1872
|
+
// `r.repo.name`). A busy issue with no resolvable run row occupies the
|
|
1873
|
+
// sentinel "" instead — matching every repo, failing toward holding.
|
|
1874
|
+
const repo = store.latestRun(project.name, issue)?.repo ?? "";
|
|
1875
|
+
const siblings = occupiedParents.get(parent);
|
|
1876
|
+
if (siblings === undefined) {
|
|
1877
|
+
occupiedParents.set(parent, new Map([[repo, issue]]));
|
|
1878
|
+
} else if (!siblings.has(repo) && !siblings.has("")) {
|
|
1879
|
+
siblings.set(repo, issue);
|
|
1845
1880
|
}
|
|
1846
1881
|
} catch (err) {
|
|
1847
1882
|
log(`#${issue} parent lookup failed while seeding epic occupancy (${errText(err)})`);
|
|
@@ -1941,10 +1976,13 @@ export async function admitCandidates(
|
|
|
1941
1976
|
continue;
|
|
1942
1977
|
}
|
|
1943
1978
|
|
|
1944
|
-
// Soft concurrency per epic: at most one in-flight child of
|
|
1945
|
-
//
|
|
1946
|
-
//
|
|
1947
|
-
//
|
|
1979
|
+
// Soft concurrency per epic, per repository: at most one in-flight child of
|
|
1980
|
+
// a given parent in each repo. Children of one epic in *different* repos
|
|
1981
|
+
// parallelise freely — `repo-active` / `maxConcurrentWorkersPerRepo` owns
|
|
1982
|
+
// the same-repo collision domain (#197). The "" sentinel matches every
|
|
1983
|
+
// repo. No parent means today's concurrent admission. Cheap local filters
|
|
1984
|
+
// already ran; this sits before the open-PR API call so a held sibling
|
|
1985
|
+
// frees the slot for unrelated work without spending a closers query.
|
|
1948
1986
|
let parent: number | undefined;
|
|
1949
1987
|
try {
|
|
1950
1988
|
parent = await resolveParent(issue);
|
|
@@ -1954,10 +1992,11 @@ export async function admitCandidates(
|
|
|
1954
1992
|
continue;
|
|
1955
1993
|
}
|
|
1956
1994
|
if (parent !== undefined) {
|
|
1957
|
-
const
|
|
1995
|
+
const occupied = occupiedParents.get(parent);
|
|
1996
|
+
const blocker = occupied?.get(r.repo.name) ?? occupied?.get("");
|
|
1958
1997
|
if (blocker !== undefined) {
|
|
1959
1998
|
hold(issue, "sibling-active");
|
|
1960
|
-
log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent}`);
|
|
1999
|
+
log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent} in ${r.repo.name}`);
|
|
1961
2000
|
continue;
|
|
1962
2001
|
}
|
|
1963
2002
|
}
|
|
@@ -2028,7 +2067,17 @@ export async function admitCandidates(
|
|
|
2028
2067
|
|
|
2029
2068
|
admitted.push({ r, attempt: priorRuns + 1 });
|
|
2030
2069
|
liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
|
|
2031
|
-
if (parent !== undefined)
|
|
2070
|
+
if (parent !== undefined) {
|
|
2071
|
+
// Extend the epic's occupancy under this repo (slot empty by construction
|
|
2072
|
+
// here — the gate above would have held the candidate otherwise) so a
|
|
2073
|
+
// same-repo sibling later in this pass does not clear the gate (#197).
|
|
2074
|
+
let siblings = occupiedParents.get(parent);
|
|
2075
|
+
if (siblings === undefined) {
|
|
2076
|
+
siblings = new Map();
|
|
2077
|
+
occupiedParents.set(parent, siblings);
|
|
2078
|
+
}
|
|
2079
|
+
if (!siblings.has(r.repo.name)) siblings.set(r.repo.name, issue);
|
|
2080
|
+
}
|
|
2032
2081
|
}
|
|
2033
2082
|
|
|
2034
2083
|
return { admitted, holds };
|
|
@@ -2124,6 +2173,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2124
2173
|
log(`label reconcile failed: ${errText(err)}`);
|
|
2125
2174
|
}
|
|
2126
2175
|
|
|
2176
|
+
// Drain the label projection outbox (#201). The maintenance phases above may
|
|
2177
|
+
// have enqueued ops (settlement releases, recovery requeues, reconciles);
|
|
2178
|
+
// each due op is applied now — or deferred with backoff for the next tick —
|
|
2179
|
+
// before the queue is read, so dispatch sees labels converging on what the
|
|
2180
|
+
// store decided. A refusing tracker defers ops instead of taking the tick
|
|
2181
|
+
// down.
|
|
2182
|
+
try {
|
|
2183
|
+
await projectLabels(d.store, d.tracker, d.project);
|
|
2184
|
+
} catch (err) {
|
|
2185
|
+
log(`label projection failed: ${errText(err)}`);
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2127
2188
|
// Ledger maintenance, above the pause gate for the same reason the stall watch
|
|
2128
2189
|
// is: a paused fleet still owes its operator the questions it asked, and a
|
|
2129
2190
|
// condition that came true while dispatch was parked is exactly the thing the
|
|
@@ -2216,7 +2277,17 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2216
2277
|
new Set(ready.map((issue) => issue.number)),
|
|
2217
2278
|
d.cleanup ?? { next: 0 },
|
|
2218
2279
|
);
|
|
2219
|
-
|
|
2280
|
+
// Label-projection overlay (#201): an issue whose outbox ops have not
|
|
2281
|
+
// reached GitHub yet is judged on what its labels *will* be. A pending
|
|
2282
|
+
// state-label removal stops a stale GitHub label from blocking redispatch,
|
|
2283
|
+
// and a pending queue-label removal drops the issue out of eligibility even
|
|
2284
|
+
// though the label is still physically present. isEligible stays pure; the
|
|
2285
|
+
// overlay happens here.
|
|
2286
|
+
const effective = ready.map((issue) => {
|
|
2287
|
+
const pending = store.pendingLabelOpsFor(project.name, issue.number);
|
|
2288
|
+
return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
|
|
2289
|
+
});
|
|
2290
|
+
const { routed, unroutable } = route(effective, project);
|
|
2220
2291
|
const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
|
|
2221
2292
|
issue: u.issue.number,
|
|
2222
2293
|
reason: `unroutable:${u.reason}`,
|
|
@@ -2300,6 +2371,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
|
|
|
2300
2371
|
(a) => handleIssue(d, a.r, a.attempt),
|
|
2301
2372
|
workers,
|
|
2302
2373
|
);
|
|
2374
|
+
|
|
2375
|
+
// Post-admission flush: freshly claimed runs enqueued their in-progress
|
|
2376
|
+
// label inside `handleIssue`; applying it now means the guard label lands on
|
|
2377
|
+
// GitHub within the same tick on the healthy path, not five minutes later
|
|
2378
|
+
// (#201).
|
|
2379
|
+
try {
|
|
2380
|
+
await projectLabels(store, d.tracker, project);
|
|
2381
|
+
} catch (err) {
|
|
2382
|
+
log(`label projection failed: ${errText(err)}`);
|
|
2383
|
+
}
|
|
2303
2384
|
}
|
|
2304
2385
|
|
|
2305
2386
|
// --------------------------------------------------------------- read-only views
|
|
@@ -2468,6 +2549,21 @@ export interface StatusSnapshot {
|
|
|
2468
2549
|
* (#188).
|
|
2469
2550
|
*/
|
|
2470
2551
|
github?: RateLimitStatus;
|
|
2552
|
+
/**
|
|
2553
|
+
* Observed GitHub rate-limit refusals within the last five minutes, and the
|
|
2554
|
+
* daemon's tracked per-source `gh` call counts for the UTC day. Unlike
|
|
2555
|
+
* `github`, which a caller polls, these are written by the tracker's hooks —
|
|
2556
|
+
* the polled budget sat beside what actually happened (#198).
|
|
2557
|
+
*/
|
|
2558
|
+
ghRefusals?: { count: number; latestAt?: number };
|
|
2559
|
+
ghCallsToday?: readonly { source: string; calls: number }[];
|
|
2560
|
+
/**
|
|
2561
|
+
* Label-projection ops still owed to the tracker (#201). Present only while
|
|
2562
|
+
* one is pending: GitHub has not yet converged on what the store decided —
|
|
2563
|
+
* a refused or deferred label write is exactly the state an operator should
|
|
2564
|
+
* see rather than a silent gap.
|
|
2565
|
+
*/
|
|
2566
|
+
labelOps?: { pending: number; oldestAgeMs: number };
|
|
2471
2567
|
}
|
|
2472
2568
|
|
|
2473
2569
|
/** Builds a status reading from an already-open store. Long-lived operator
|
|
@@ -2481,6 +2577,8 @@ export function statusSnapshotFromStore(
|
|
|
2481
2577
|
): StatusSnapshot {
|
|
2482
2578
|
const since = startOfToday();
|
|
2483
2579
|
const dispatch = store.latestDispatch(p.name);
|
|
2580
|
+
const labelOpsPending = store.countPendingLabelOps(p.name);
|
|
2581
|
+
const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
|
|
2484
2582
|
return {
|
|
2485
2583
|
project: p.name,
|
|
2486
2584
|
configPath: configPath(),
|
|
@@ -2497,6 +2595,13 @@ export function statusSnapshotFromStore(
|
|
|
2497
2595
|
spendTodayUsd: store.spendSince(p.name, since),
|
|
2498
2596
|
...(dispatch === undefined ? {} : { dispatch }),
|
|
2499
2597
|
...(planUsage === undefined ? {} : { planUsage }),
|
|
2598
|
+
// Written by the tracker's hooks rather than polled, so the renderer does
|
|
2599
|
+
// not re-read GitHub to know it is being refused (#198).
|
|
2600
|
+
ghRefusals: store.ghRefusalsSince?.(Date.now() - 5 * 60_000),
|
|
2601
|
+
ghCallsToday: store.ghCallsToday?.(utcDay()),
|
|
2602
|
+
...(labelOpsPending === 0 || oldestLabelOpAt === undefined
|
|
2603
|
+
? {}
|
|
2604
|
+
: { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
|
|
2500
2605
|
};
|
|
2501
2606
|
}
|
|
2502
2607
|
|
|
@@ -2829,11 +2934,11 @@ async function recoverRun(
|
|
|
2829
2934
|
const inProgress = project.stateLabels.inProgress;
|
|
2830
2935
|
|
|
2831
2936
|
if (recovery === "settle") {
|
|
2832
|
-
//
|
|
2833
|
-
//
|
|
2834
|
-
//
|
|
2835
|
-
//
|
|
2836
|
-
|
|
2937
|
+
// Enqueue the release with the terminal write (see `settlePushedGreen`):
|
|
2938
|
+
// the outbox keeps the label and the row one fact, so a tracker refusal
|
|
2939
|
+
// can no longer strand `agent:in-progress` with nothing left to retry it
|
|
2940
|
+
// (#18, #201).
|
|
2941
|
+
releaseInProgress(d, run.issue, `PR merged: ${evidence}`);
|
|
2837
2942
|
store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
|
|
2838
2943
|
log(`#${run.issue} settled from ${cls}: ${evidence}`);
|
|
2839
2944
|
return;
|
|
@@ -2859,13 +2964,15 @@ async function recoverRun(
|
|
|
2859
2964
|
// `merge-conflict`: the branch is retained and its PR is open, so #50's
|
|
2860
2965
|
// continuation guard admits it and the next tick briefs a rebase.
|
|
2861
2966
|
//
|
|
2862
|
-
// The
|
|
2863
|
-
//
|
|
2864
|
-
// `
|
|
2865
|
-
//
|
|
2866
|
-
//
|
|
2867
|
-
//
|
|
2868
|
-
|
|
2967
|
+
// The outbox makes the retry contract one-sided: the swap is enqueued — a
|
|
2968
|
+
// durable local write that cannot fail on the tracker — before
|
|
2969
|
+
// `recoveredAt` is written, so the row can never again be taken out of
|
|
2970
|
+
// `runsNeedingClassification` with its label swap still owed. That was
|
|
2971
|
+
// the defect 0.4.4 claimed to have fixed and did not, for this one
|
|
2972
|
+
// recovery; the projector retries until the tracker takes the swap, and
|
|
2973
|
+
// while it is pending the eligibility overlay keeps the issue coherent
|
|
2974
|
+
// (#201).
|
|
2975
|
+
swapToQueue(d, run.issue, inProgress);
|
|
2869
2976
|
store.updateRun(run.id, {
|
|
2870
2977
|
state: "killed",
|
|
2871
2978
|
lastError:
|
|
@@ -2906,7 +3013,7 @@ async function recoverRun(
|
|
|
2906
3013
|
return;
|
|
2907
3014
|
}
|
|
2908
3015
|
const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
|
|
2909
|
-
|
|
3016
|
+
swapToQueue(d, run.issue, label);
|
|
2910
3017
|
store.updateRun(run.id, { recoveredAt: Date.now() });
|
|
2911
3018
|
log(`#${run.issue} requeued from ${cls}: ${evidence}`);
|
|
2912
3019
|
return;
|
|
@@ -2972,26 +3079,22 @@ async function recoverRun(
|
|
|
2972
3079
|
}
|
|
2973
3080
|
|
|
2974
3081
|
/**
|
|
2975
|
-
*
|
|
3082
|
+
* Enqueue a state-label → queue-label swap for projection (#201).
|
|
2976
3083
|
*
|
|
2977
|
-
*
|
|
2978
|
-
*
|
|
2979
|
-
*
|
|
2980
|
-
*
|
|
3084
|
+
* The swap is two ops in id order — remove first, then add — which is the
|
|
3085
|
+
* atomicity the projector guarantees: the issue never sits newly eligible
|
|
3086
|
+
* without a queue label on its way back, and the add never lands before the
|
|
3087
|
+
* remove when GitHub fails between them. Enqueueing is a durable local write
|
|
3088
|
+
* that cannot fail on the tracker, so the caller records its recovery
|
|
3089
|
+
* immediately and the projector retries the swap until the tracker takes it —
|
|
3090
|
+
* that closes the 0.4.4 hole where a refused label swap stranded the row
|
|
3091
|
+
* permanently under a log line promising a retry.
|
|
2981
3092
|
*/
|
|
2982
|
-
|
|
2983
|
-
d
|
|
2984
|
-
|
|
2985
|
-
|
|
2986
|
-
)
|
|
2987
|
-
try {
|
|
2988
|
-
await d.tracker.removeLabel(issue, label);
|
|
2989
|
-
await d.tracker.addLabel(issue, d.project.queueLabel);
|
|
2990
|
-
return true;
|
|
2991
|
-
} catch (err) {
|
|
2992
|
-
log(`#${issue} could not be requeued (${errText(err)}) — retrying next tick`);
|
|
2993
|
-
return false;
|
|
2994
|
-
}
|
|
3093
|
+
function swapToQueue(d: Pick<Deps, "project" | "store">, issue: number, label: string): void {
|
|
3094
|
+
d.store.enqueueLabelOps(d.project.name, [
|
|
3095
|
+
{ issue, op: "remove", label },
|
|
3096
|
+
{ issue, op: "add", label: d.project.queueLabel },
|
|
3097
|
+
]);
|
|
2995
3098
|
}
|
|
2996
3099
|
|
|
2997
3100
|
/** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
|
|
@@ -3021,13 +3124,11 @@ export async function reconcileStaleLabels(d: Deps): Promise<void> {
|
|
|
3021
3124
|
for (const issue of carrying) {
|
|
3022
3125
|
if (issue.state === "closed") {
|
|
3023
3126
|
// Never retain an `agent:*` label on a closed issue: the work is done by
|
|
3024
|
-
// some route, and the label only makes the board lie about it.
|
|
3025
|
-
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
}
|
|
3029
|
-
log(`#${issue.number} could not drop ${label} (${errText(err)}) — retrying next tick`);
|
|
3030
|
-
}
|
|
3127
|
+
// some route, and the label only makes the board lie about it. Enqueue
|
|
3128
|
+
// rather than call — a refused write must not lose the decision; the
|
|
3129
|
+
// projector retries the removal until the tracker takes it (#201).
|
|
3130
|
+
store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
|
|
3131
|
+
log(`#${issue.number} reconciled: closed issue no longer carries ${label} (queued)`);
|
|
3031
3132
|
continue;
|
|
3032
3133
|
}
|
|
3033
3134
|
|
|
@@ -3038,8 +3139,8 @@ export async function reconcileStaleLabels(d: Deps): Promise<void> {
|
|
|
3038
3139
|
const key = `${project.name}:superseded:${issue.number}`;
|
|
3039
3140
|
if (store.wasNotified(key)) continue;
|
|
3040
3141
|
const list = children.map((c) => `#${c.number}`).join(", ");
|
|
3142
|
+
store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
|
|
3041
3143
|
try {
|
|
3042
|
-
await tracker.removeLabel(issue.number, label);
|
|
3043
3144
|
await tracker.comment(
|
|
3044
3145
|
issue.number,
|
|
3045
3146
|
`superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
|
|
@@ -3048,7 +3149,9 @@ export async function reconcileStaleLabels(d: Deps): Promise<void> {
|
|
|
3048
3149
|
store.markNotified(key);
|
|
3049
3150
|
log(`#${issue.number} reconciled: superseded by ${list}`);
|
|
3050
3151
|
} catch (err) {
|
|
3051
|
-
|
|
3152
|
+
// The label removal is already queued and will land regardless; the
|
|
3153
|
+
// comment is the only half that can fail here (#201).
|
|
3154
|
+
log(`#${issue.number} could not comment the superseded note (${errText(err)}) — retrying next tick`);
|
|
3052
3155
|
}
|
|
3053
3156
|
}
|
|
3054
3157
|
}
|
|
@@ -3124,7 +3227,19 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
|
|
|
3124
3227
|
const project = findProject(cfg, o.project);
|
|
3125
3228
|
const caps = resolveCaps(project, cfg.defaults);
|
|
3126
3229
|
const store = openStore(dbPath());
|
|
3127
|
-
|
|
3230
|
+
// The tracker's single `gh` funnel is bound to the store so the operator sees
|
|
3231
|
+
// observed truth (#198): every spawn is counted per UTC day, and every
|
|
3232
|
+
// rate-limit refusal is recorded for `status`'s 5m window. Board's ad-hoc
|
|
3233
|
+
// trackers and the polled `fetchRateLimit` probe are deliberately not bound —
|
|
3234
|
+
// this row is the daemon's own traffic.
|
|
3235
|
+
const tracker = makeTracker(project, undefined, {
|
|
3236
|
+
onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
|
|
3237
|
+
onRefusal: (at) => store.recordGhRefusal?.(at),
|
|
3238
|
+
// A conditional 304 revalidation is a spawn but not a billed read (#203);
|
|
3239
|
+
// counted separately so `status`'s call row keeps telling the truth once
|
|
3240
|
+
// most spawns are free revalidations.
|
|
3241
|
+
onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
|
|
3242
|
+
});
|
|
3128
3243
|
|
|
3129
3244
|
// #126's transport, stated at startup rather than guessed at first use. The
|
|
3130
3245
|
// banner names what this host can actually enforce — whether the kernel will
|