omp-conductor 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/daemon.ts CHANGED
@@ -32,11 +32,12 @@ import { startOrchestrator } from "./orchestrator.ts";
32
32
  import type { OrchestratorHandle } from "./orchestrator.ts";
33
33
  import { createReportOutbox, formatOpenReports } from "./reports.ts";
34
34
  import { recordReleaseBlock } from "./release-policy.ts";
35
- import { branchName, route } from "./routing.ts";
35
+ import { branchName, effectiveLabels, route } from "./routing.ts";
36
36
  import type { Routed, UnroutableReason } from "./routing.ts";
37
37
  import { evaluateDecisionConditions, probeNpmVersion, probeRateLimitReset } from "./decisions.ts";
38
38
  import { classifyRun, type ClassifyFacts } from "./failure-class.ts";
39
- import { dbPath, openStore } from "./store.ts";
39
+ import { projectLabels } from "./label-projection.ts";
40
+ import { dbPath, openStore, utcDay } from "./store.ts";
40
41
  import { makeTracker, type RateLimitStatus } from "./tracker/github.ts";
41
42
  import { RELEASE_SHAPES } from "./types.ts";
42
43
  import type {
@@ -515,13 +516,17 @@ function acceptanceCriteria(issue: ReadyIssue): string {
515
516
  }
516
517
 
517
518
  /**
518
- * Add the new label before dropping the old one. The reverse order leaves a
519
- * window where the issue carries no state label at all, which is exactly the
520
- * shape `isEligible` treats as fresh work.
519
+ * Record a state-label swap for projection (#201). The add enqueues before the
520
+ * remove the reverse order would leave a window where the issue carries no
521
+ * state label at all, which is exactly the shape `isEligible` treats as fresh
522
+ * work. Synchronous: enqueueing is a local store write and cannot fail on the
523
+ * tracker; the projector applies the pair in order and retries on refusal.
521
524
  */
522
- async function swapLabel(tracker: Tracker, issue: number, from: string, to: string): Promise<void> {
523
- await tracker.addLabel(issue, to);
524
- await tracker.removeLabel(issue, from);
525
+ function swapLabel(store: Store, projectName: string, issue: number, from: string, to: string): void {
526
+ store.enqueueLabelOps(projectName, [
527
+ { issue, op: "add", label: to },
528
+ { issue, op: "remove", label: from },
529
+ ]);
525
530
  }
526
531
 
527
532
  /**
@@ -855,13 +860,18 @@ export async function verifyPushedGreenClaim(
855
860
  */
856
861
  export async function collectSettlementFlags(
857
862
  tracker: Pick<Tracker, "prDiff">,
858
- claim: { prUrl?: string; report: string; issueText: string },
863
+ claim: { prUrl?: string; report: string; priorReports?: readonly string[]; issueText: string },
859
864
  ): Promise<{ flags: SettlementFlag[]; truncated: boolean } | undefined> {
860
865
  if (claim.prUrl === undefined) return undefined;
861
866
  const diff = await tracker.prDiff(claim.prUrl);
862
867
  if (diff === undefined) return undefined;
863
868
  return {
864
- flags: analyseSettlement({ report: claim.report, issueText: claim.issueText, diff }),
869
+ flags: analyseSettlement({
870
+ report: claim.report,
871
+ issueText: claim.issueText,
872
+ diff,
873
+ priorReports: claim.priorReports,
874
+ }),
865
875
  truncated: diff.truncated,
866
876
  };
867
877
  }
@@ -938,12 +948,17 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
938
948
  };
939
949
 
940
950
  try {
941
- // Claim on the tracker FIRST, before any local work. The label not the
942
- // store — is the crash-safe guard against double dispatch: if this process
943
- // dies mid-run, the next daemon sees the label, `isEligible` filters the
944
- // issue out, and a human decides what to do with the orphan.
945
- await tracker.addLabel(issue, inProgress);
946
- claimed = true;
951
+ // Claim on the STORE first, before anything that can fail. The run row
952
+ // not the label — is the crash-safe guard against double dispatch: rows
953
+ // are local, written before any network call, and the startup orphan
954
+ // sweep marks process-less claimed/running rows `orphaned`, so a daemon
955
+ // dying mid-claim leaves a row a human can triage instead of a label only
956
+ // the orchestrator may touch. The in-progress label is a write-behind
957
+ // projection of that row: enqueued here, flushed by the post-admission
958
+ // projection pass in the same tick on the healthy path, and — while
959
+ // pending — kept effective off the ready set by the routing overlay, so a
960
+ // crash between the row and its label cannot double-dispatch either
961
+ // (#201).
947
962
  // Read before this attempt's own row exists, so `latestRun` still means the
948
963
  // attempt whose work this one inherits.
949
964
  const priorSalvage = store.latestRun(project.name, issue)?.salvageSha;
@@ -963,6 +978,8 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
963
978
  startedAt: Date.now(),
964
979
  });
965
980
  const runId = run.id;
981
+ store.enqueueLabelOps(project.name, [{ issue, op: "add", label: inProgress }]);
982
+ claimed = true;
966
983
  turnLimit = d.turnLimits.open(project.name, issue, runId, caps.workerMaxTurns);
967
984
 
968
985
  // A run's tree is <workspaceRoot>/<issue> and addRunRepo refuses to reuse
@@ -1102,6 +1119,15 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1102
1119
  ? await collectSettlementFlags(tracker, {
1103
1120
  prUrl: result.prUrl,
1104
1121
  report: result.report,
1122
+ // Earlier attempts' reports pool as disclosure coverage (#199). The
1123
+ // current row has not settled yet, so this is exactly the prior set;
1124
+ // the strict `< attempt` guard keeps an already-persisted copy of
1125
+ // this attempt out just in case. Pre-#199 rows have NULL reports and
1126
+ // contribute nothing — today's behaviour.
1127
+ priorReports: store
1128
+ .attemptReports(project.name, issue)
1129
+ .filter((r) => r.attempt < attempt)
1130
+ .map((r) => r.report),
1105
1131
  issueText: `${r.issue.title}\n${r.issue.body}`,
1106
1132
  })
1107
1133
  : undefined;
@@ -1110,7 +1136,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1110
1136
  else if (audit.truncated) log(`#${issue} settlement audit read a truncated PR diff`);
1111
1137
  }
1112
1138
  const auditLines =
1113
- audit === undefined ? [] : formatSettlementFlags(audit.flags, { truncated: audit.truncated });
1139
+ audit === undefined
1140
+ ? []
1141
+ : formatSettlementFlags(audit.flags, { truncated: audit.truncated, attempts: attempt });
1114
1142
 
1115
1143
  const finalReport = [
1116
1144
  ...(verified.reason === undefined ? [] : [verified.reason, ""]),
@@ -1160,6 +1188,10 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1160
1188
  prUrl: result.prUrl,
1161
1189
  headSha: result.headSha,
1162
1190
  sessionFile: result.sessionFile,
1191
+ // Every terminal state persists the worker's report, not just a green
1192
+ // push: the report of a killed attempt is exactly the one a later
1193
+ // continuation must pool its disclosures from (#199).
1194
+ report: result.report,
1163
1195
  ...(verified.reason === undefined ? {} : { lastError: verified.reason }),
1164
1196
  ...settlement?.patch,
1165
1197
  ...(audit === undefined || audit.flags.length === 0
@@ -1170,7 +1202,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1170
1202
  const salvaged = settlement?.lines ?? [];
1171
1203
 
1172
1204
  if (state === "blocked") {
1173
- await swapLabel(tracker, issue, inProgress, project.stateLabels.blocked);
1205
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.blocked);
1174
1206
  await safeEscalate(d, {
1175
1207
  tier: 1,
1176
1208
  project: project.name,
@@ -1193,8 +1225,12 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1193
1225
  });
1194
1226
 
1195
1227
  if (continueTurns) {
1196
- await tracker.removeLabel(issue, inProgress);
1197
- await tracker.addLabel(issue, project.queueLabel);
1228
+ // Requeue as one ordered pair: the in-progress removal before the
1229
+ // queue add, exactly the order the projector will apply them in (#201).
1230
+ store.enqueueLabelOps(project.name, [
1231
+ { issue, op: "remove", label: inProgress },
1232
+ { issue, op: "add", label: project.queueLabel },
1233
+ ]);
1198
1234
  log(
1199
1235
  `#${issue} turns-cap on run ${attempt}, continuation ` +
1200
1236
  `${continuation}/${caps.maxContinuationsPerIssue} — salvaged and re-queued`,
@@ -1218,7 +1254,7 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1218
1254
  ].join("\n"),
1219
1255
  });
1220
1256
  } else {
1221
- await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
1257
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
1222
1258
  await safeEscalate(d, {
1223
1259
  tier: 1,
1224
1260
  project: project.name,
@@ -1316,11 +1352,9 @@ async function handleIssue(d: Deps, r: Routed, attempt: number): Promise<void> {
1316
1352
  if (claimed) {
1317
1353
  // Leaving the issue stuck as in-progress would hide it from both the
1318
1354
  // queue and the human, so relabel even on the error path.
1319
- try {
1320
- await swapLabel(tracker, issue, inProgress, project.stateLabels.failed);
1321
- } catch (relabelErr) {
1322
- log(`#${issue} could not be relabelled: ${errText(relabelErr)}`);
1323
- }
1355
+ // The failure-path relabel cannot throw: it is a local outbox write, and
1356
+ // the projector retries until the tracker takes it (#201).
1357
+ swapLabel(store, project.name, issue, inProgress, project.stateLabels.failed);
1324
1358
  }
1325
1359
 
1326
1360
  const salvaged = settlement?.lines ?? [];
@@ -1394,46 +1428,35 @@ export function settlementFor(pr: PrState | undefined, prUrl: string): Settlemen
1394
1428
  }
1395
1429
 
1396
1430
  /**
1397
- * Drops the in-progress label from an issue whose run is provably over.
1431
+ * Records the in-progress label's release as a projection op (#201).
1398
1432
  *
1399
1433
  * Settlement used to write only half of what it knew. On 2026-08-09 that cost
1400
1434
  * the reference fleet two issues in one night: veltro#331 settled to `failed`
1401
1435
  * at 23:53Z once the orchestrator closed veltro#332 unmerged, and veltro#344
1402
1436
  * settled to `merged` at 05:54Z once chad#452 squash-merged. Both rows left the
1403
1437
  * active set correctly; an authoritative `gh issue view` on each afterwards
1404
- * still showed `agent:in-progress`. `routing.isEligible` rejects any issue
1405
- * carrying a state label, the composed brief forbids the orchestrator from
1406
- * hand-editing one, and `unblock` refused to clear that particular label — so
1407
- * both issues were permanently unclaimable with no supported way back (#18).
1438
+ * still showed `agent:in-progress` permanently unclaimable with no supported
1439
+ * way back (#18).
1408
1440
  *
1409
- * Never throws, and reports whether the label is provably gone, because the
1410
- * caller has to decide what to write to the store on the strength of it. A
1411
- * sweep must not lose the rest of its rows to one unreachable tracker, and it
1412
- * must not terminalise a row whose label it failed to drop: the settlement
1413
- * sweep only ever revisits `pushed-*` rows, so a row written terminal is a row
1414
- * nothing asks about again, and swallowing the failure under it would recreate
1415
- * the exact permanent-`agent:in-progress` state of #18 in the one case that
1416
- * still reaches it. Answering false instead leaves the row where the next tick
1417
- * will find it.
1441
+ * The outbox makes the row transition and the label one fact again: the
1442
+ * removal is enqueued in the same breath as the row is terminalised, the
1443
+ * projector applies it with unbounded retry, and while it is pending the
1444
+ * eligibility overlay treats the label as already gone. A tracker that refuses
1445
+ * the write (403, rate limit) can no longer strand the row that is `#184`
1446
+ * and `#198` closed. No `pushed-*` row is ever written terminal with its
1447
+ * label release owed but unrecorded, because enqueueing is a local store write
1448
+ * that cannot fail on the tracker.
1418
1449
  *
1419
- * Removing a label the issue does not carry is success, not failure: the GitHub
1420
- * adapter treats an absent label as a no-op, so false means the tracker could
1421
- * not be reached or refused — a condition that passes.
1450
+ * Synchronous. The op is durable the moment this returns.
1422
1451
  */
1423
- export async function releaseInProgress(
1424
- d: Pick<Deps, "project" | "tracker">,
1452
+ export function releaseInProgress(
1453
+ d: Pick<Deps, "project" | "store">,
1425
1454
  issue: number,
1426
1455
  why: string,
1427
- ): Promise<boolean> {
1456
+ ): void {
1428
1457
  const label = d.project.stateLabels.inProgress;
1429
- try {
1430
- await d.tracker.removeLabel(issue, label);
1431
- log(`#${issue} released ${label}: ${why}`);
1432
- return true;
1433
- } catch (err) {
1434
- log(`#${issue} could not release ${label} (${errText(err)}) — ${why}; retrying next tick`);
1435
- return false;
1436
- }
1458
+ d.store.enqueueLabelOps(d.project.name, [{ issue, op: "remove", label }]);
1459
+ log(`#${issue} released ${label} (queued): ${why}`);
1437
1460
  }
1438
1461
 
1439
1462
  /**
@@ -1507,15 +1530,15 @@ export async function settlePushedGreen(
1507
1530
 
1508
1531
  const settlement = settlementFor(pr, run.prUrl);
1509
1532
  if (settlement !== undefined) {
1510
- // Label first, row second, and the order is the whole safety argument.
1511
- // The sweep only ever revisits `pushed-*` rows, so writing the terminal
1512
- // state first would put this row beyond every later tick and a tracker
1513
- // that failed on the label in that instant would strand
1514
- // `agent:in-progress` permanently, which is #18 again in the one window
1515
- // still able to reach it. Leaving the row `pushed-*` costs a stale active
1516
- // row until the tracker answers, and the busy set keeps the issue
1517
- // occupied meanwhile, so nothing can be dispatched onto it in between.
1518
- if (!(await releaseInProgress(d, run.issue, settlement.reason))) continue;
1533
+ // The label removal and the terminal row are one fact again (#201): the
1534
+ // release is enqueued a durable local write that cannot fail on the
1535
+ // tracker in the same breath as the row is terminalised, so there is
1536
+ // no window in which a row beyond every later tick still owes its label.
1537
+ // The projector applies it with retry; while pending, the eligibility
1538
+ // overlay treats the label as already gone, so #18's
1539
+ // permanent-`agent:in-progress` cannot re-form even when GitHub refuses
1540
+ // the write.
1541
+ releaseInProgress(d, run.issue, settlement.reason);
1519
1542
  const patch: Partial<RunRecord> = { state: settlement.state, endedAt: Date.now() };
1520
1543
  if (settlement.state === "failed") patch.lastError = settlement.reason;
1521
1544
  store.updateRun(run.id, patch);
@@ -1536,11 +1559,11 @@ export async function settlePushedGreen(
1536
1559
  store.updateRun(run.id, { state: "pushed-green", lastError: undefined });
1537
1560
  log(`#${run.issue} checks settled: ${verification.reason}`);
1538
1561
  } else if (verification.status === "failed") {
1539
- // Equally terminal, so the same label-first order for the same reason:
1540
- // this row is about to leave the sweep's reach. The green branch above
1541
- // releases nothing — that row is still awaiting a merge, and its live PR
1542
- // is exactly the work the label must keep guarding.
1543
- if (!(await releaseInProgress(d, run.issue, verification.reason))) continue;
1562
+ // Equally terminal, so the release is enqueued before the row writes,
1563
+ // for the same reason as the settlement branch above (see there). The
1564
+ // green branch releases nothing — that row is still awaiting a merge,
1565
+ // and its live PR is exactly the work the label must keep guarding.
1566
+ releaseInProgress(d, run.issue, verification.reason);
1544
1567
  store.updateRun(run.id, { state: "failed", lastError: verification.reason });
1545
1568
  log(`#${run.issue} checks failed: ${verification.reason}`);
1546
1569
  } else {
@@ -1822,10 +1845,12 @@ export async function admitCandidates(
1822
1845
  return { admitted: [], holds };
1823
1846
  }
1824
1847
 
1825
- // parent -> blocking issue. Seeded from active runs (including pushed-green),
1826
- // then extended by candidates admitted earlier in this same pass so two
1827
- // siblings never both clear the gate in one tick.
1828
- const occupiedParents = new Map<number, number>();
1848
+ // parent -> repo name -> blocking issue. Seeded from active runs (including
1849
+ // pushed-green), then extended by candidates admitted earlier in this same
1850
+ // pass so two siblings of one epic never both clear the gate in one tick.
1851
+ // A busy issue whose run row cannot be resolved occupies the sentinel repo
1852
+ // "" — treated as matching every repo, failing toward holding (#197).
1853
+ const occupiedParents = new Map<number, Map<string, number>>();
1829
1854
  const parentCache = new Map<number, number | undefined>();
1830
1855
 
1831
1856
  const resolveParent = async (issue: number): Promise<number | undefined> => {
@@ -1840,8 +1865,18 @@ export async function admitCandidates(
1840
1865
  for (const issue of busyIssues) {
1841
1866
  try {
1842
1867
  const parent = await resolveParent(issue);
1843
- if (parent !== undefined && !occupiedParents.has(parent)) {
1844
- occupiedParents.set(parent, issue);
1868
+ if (parent === undefined) continue;
1869
+ // The runs table records which repo each attempt worked in, and sibling
1870
+ // holds are now per-repo, so a busy child only occupies its epic under
1871
+ // that repo's name (same spelling as `createRun` writes from
1872
+ // `r.repo.name`). A busy issue with no resolvable run row occupies the
1873
+ // sentinel "" instead — matching every repo, failing toward holding.
1874
+ const repo = store.latestRun(project.name, issue)?.repo ?? "";
1875
+ const siblings = occupiedParents.get(parent);
1876
+ if (siblings === undefined) {
1877
+ occupiedParents.set(parent, new Map([[repo, issue]]));
1878
+ } else if (!siblings.has(repo) && !siblings.has("")) {
1879
+ siblings.set(repo, issue);
1845
1880
  }
1846
1881
  } catch (err) {
1847
1882
  log(`#${issue} parent lookup failed while seeding epic occupancy (${errText(err)})`);
@@ -1941,10 +1976,13 @@ export async function admitCandidates(
1941
1976
  continue;
1942
1977
  }
1943
1978
 
1944
- // Soft concurrency per epic: at most one in-flight child of a given parent.
1945
- // No parent means today's concurrent admission. Cheap local filters already
1946
- // ran; this sits before the open-PR API call so a held sibling frees the
1947
- // slot for unrelated work without spending a closers query.
1979
+ // Soft concurrency per epic, per repository: at most one in-flight child of
1980
+ // a given parent in each repo. Children of one epic in *different* repos
1981
+ // parallelise freely `repo-active` / `maxConcurrentWorkersPerRepo` owns
1982
+ // the same-repo collision domain (#197). The "" sentinel matches every
1983
+ // repo. No parent means today's concurrent admission. Cheap local filters
1984
+ // already ran; this sits before the open-PR API call so a held sibling
1985
+ // frees the slot for unrelated work without spending a closers query.
1948
1986
  let parent: number | undefined;
1949
1987
  try {
1950
1988
  parent = await resolveParent(issue);
@@ -1954,10 +1992,11 @@ export async function admitCandidates(
1954
1992
  continue;
1955
1993
  }
1956
1994
  if (parent !== undefined) {
1957
- const blocker = occupiedParents.get(parent);
1995
+ const occupied = occupiedParents.get(parent);
1996
+ const blocker = occupied?.get(r.repo.name) ?? occupied?.get("");
1958
1997
  if (blocker !== undefined) {
1959
1998
  hold(issue, "sibling-active");
1960
- log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent}`);
1999
+ log(`#${issue} skipped: sibling #${blocker} in flight under epic #${parent} in ${r.repo.name}`);
1961
2000
  continue;
1962
2001
  }
1963
2002
  }
@@ -2028,7 +2067,17 @@ export async function admitCandidates(
2028
2067
 
2029
2068
  admitted.push({ r, attempt: priorRuns + 1 });
2030
2069
  liveByRepo.set(r.repo.name, (liveByRepo.get(r.repo.name) ?? 0) + 1);
2031
- if (parent !== undefined) occupiedParents.set(parent, issue);
2070
+ if (parent !== undefined) {
2071
+ // Extend the epic's occupancy under this repo (slot empty by construction
2072
+ // here — the gate above would have held the candidate otherwise) so a
2073
+ // same-repo sibling later in this pass does not clear the gate (#197).
2074
+ let siblings = occupiedParents.get(parent);
2075
+ if (siblings === undefined) {
2076
+ siblings = new Map();
2077
+ occupiedParents.set(parent, siblings);
2078
+ }
2079
+ if (!siblings.has(r.repo.name)) siblings.set(r.repo.name, issue);
2080
+ }
2032
2081
  }
2033
2082
 
2034
2083
  return { admitted, holds };
@@ -2124,6 +2173,18 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2124
2173
  log(`label reconcile failed: ${errText(err)}`);
2125
2174
  }
2126
2175
 
2176
+ // Drain the label projection outbox (#201). The maintenance phases above may
2177
+ // have enqueued ops (settlement releases, recovery requeues, reconciles);
2178
+ // each due op is applied now — or deferred with backoff for the next tick —
2179
+ // before the queue is read, so dispatch sees labels converging on what the
2180
+ // store decided. A refusing tracker defers ops instead of taking the tick
2181
+ // down.
2182
+ try {
2183
+ await projectLabels(d.store, d.tracker, d.project);
2184
+ } catch (err) {
2185
+ log(`label projection failed: ${errText(err)}`);
2186
+ }
2187
+
2127
2188
  // Ledger maintenance, above the pause gate for the same reason the stall watch
2128
2189
  // is: a paused fleet still owes its operator the questions it asked, and a
2129
2190
  // condition that came true while dispatch was parked is exactly the thing the
@@ -2216,7 +2277,17 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2216
2277
  new Set(ready.map((issue) => issue.number)),
2217
2278
  d.cleanup ?? { next: 0 },
2218
2279
  );
2219
- const { routed, unroutable } = route(ready, project);
2280
+ // Label-projection overlay (#201): an issue whose outbox ops have not
2281
+ // reached GitHub yet is judged on what its labels *will* be. A pending
2282
+ // state-label removal stops a stale GitHub label from blocking redispatch,
2283
+ // and a pending queue-label removal drops the issue out of eligibility even
2284
+ // though the label is still physically present. isEligible stays pure; the
2285
+ // overlay happens here.
2286
+ const effective = ready.map((issue) => {
2287
+ const pending = store.pendingLabelOpsFor(project.name, issue.number);
2288
+ return pending.length === 0 ? issue : { ...issue, labels: effectiveLabels(issue.labels, pending) };
2289
+ });
2290
+ const { routed, unroutable } = route(effective, project);
2220
2291
  const routingHolds: AdmissionHold[] = unroutable.map((u) => ({
2221
2292
  issue: u.issue.number,
2222
2293
  reason: `unroutable:${u.reason}`,
@@ -2300,6 +2371,16 @@ export async function tick(d: Deps, workers?: WorkerPool): Promise<void> {
2300
2371
  (a) => handleIssue(d, a.r, a.attempt),
2301
2372
  workers,
2302
2373
  );
2374
+
2375
+ // Post-admission flush: freshly claimed runs enqueued their in-progress
2376
+ // label inside `handleIssue`; applying it now means the guard label lands on
2377
+ // GitHub within the same tick on the healthy path, not five minutes later
2378
+ // (#201).
2379
+ try {
2380
+ await projectLabels(store, d.tracker, project);
2381
+ } catch (err) {
2382
+ log(`label projection failed: ${errText(err)}`);
2383
+ }
2303
2384
  }
2304
2385
 
2305
2386
  // --------------------------------------------------------------- read-only views
@@ -2468,6 +2549,21 @@ export interface StatusSnapshot {
2468
2549
  * (#188).
2469
2550
  */
2470
2551
  github?: RateLimitStatus;
2552
+ /**
2553
+ * Observed GitHub rate-limit refusals within the last five minutes, and the
2554
+ * daemon's tracked per-source `gh` call counts for the UTC day. Unlike
2555
+ * `github`, which a caller polls, these are written by the tracker's hooks —
2556
+ * the polled budget sat beside what actually happened (#198).
2557
+ */
2558
+ ghRefusals?: { count: number; latestAt?: number };
2559
+ ghCallsToday?: readonly { source: string; calls: number }[];
2560
+ /**
2561
+ * Label-projection ops still owed to the tracker (#201). Present only while
2562
+ * one is pending: GitHub has not yet converged on what the store decided —
2563
+ * a refused or deferred label write is exactly the state an operator should
2564
+ * see rather than a silent gap.
2565
+ */
2566
+ labelOps?: { pending: number; oldestAgeMs: number };
2471
2567
  }
2472
2568
 
2473
2569
  /** Builds a status reading from an already-open store. Long-lived operator
@@ -2481,6 +2577,8 @@ export function statusSnapshotFromStore(
2481
2577
  ): StatusSnapshot {
2482
2578
  const since = startOfToday();
2483
2579
  const dispatch = store.latestDispatch(p.name);
2580
+ const labelOpsPending = store.countPendingLabelOps(p.name);
2581
+ const oldestLabelOpAt = store.oldestPendingLabelOpAt(p.name);
2484
2582
  return {
2485
2583
  project: p.name,
2486
2584
  configPath: configPath(),
@@ -2497,6 +2595,13 @@ export function statusSnapshotFromStore(
2497
2595
  spendTodayUsd: store.spendSince(p.name, since),
2498
2596
  ...(dispatch === undefined ? {} : { dispatch }),
2499
2597
  ...(planUsage === undefined ? {} : { planUsage }),
2598
+ // Written by the tracker's hooks rather than polled, so the renderer does
2599
+ // not re-read GitHub to know it is being refused (#198).
2600
+ ghRefusals: store.ghRefusalsSince?.(Date.now() - 5 * 60_000),
2601
+ ghCallsToday: store.ghCallsToday?.(utcDay()),
2602
+ ...(labelOpsPending === 0 || oldestLabelOpAt === undefined
2603
+ ? {}
2604
+ : { labelOps: { pending: labelOpsPending, oldestAgeMs: Date.now() - oldestLabelOpAt } }),
2500
2605
  };
2501
2606
  }
2502
2607
 
@@ -2829,11 +2934,11 @@ async function recoverRun(
2829
2934
  const inProgress = project.stateLabels.inProgress;
2830
2935
 
2831
2936
  if (recovery === "settle") {
2832
- // Label before row, copied from `settlePushedGreen` where the order is the
2833
- // whole safety argument: writing the terminal state first would put this row
2834
- // beyond every later tick, and a tracker that then failed on the label would
2835
- // strand `agent:in-progress` with nothing left to retry it (#18).
2836
- if (!(await releaseInProgress(d, run.issue, `PR merged: ${evidence}`))) return;
2937
+ // Enqueue the release with the terminal write (see `settlePushedGreen`):
2938
+ // the outbox keeps the label and the row one fact, so a tracker refusal
2939
+ // can no longer strand `agent:in-progress` with nothing left to retry it
2940
+ // (#18, #201).
2941
+ releaseInProgress(d, run.issue, `PR merged: ${evidence}`);
2837
2942
  store.updateRun(run.id, { state: "merged", endedAt: Date.now(), recoveredAt: Date.now() });
2838
2943
  log(`#${run.issue} settled from ${cls}: ${evidence}`);
2839
2944
  return;
@@ -2859,13 +2964,15 @@ async function recoverRun(
2859
2964
  // `merge-conflict`: the branch is retained and its PR is open, so #50's
2860
2965
  // continuation guard admits it and the next tick briefs a rebase.
2861
2966
  //
2862
- // The tracker write goes FIRST, and that ordering is the whole retry
2863
- // contract. Writing `recoveredAt` before the swap took the row out of
2864
- // `runsNeedingClassification` which selects on `recoveredAt IS NULL` so a
2865
- // refused label swap stranded the row permanently under a log line promising
2866
- // a retry. That was the defect 0.4.4 claimed to have fixed and did not, for
2867
- // this one recovery.
2868
- if (!(await swapToQueue(d, run.issue, inProgress))) return;
2967
+ // The outbox makes the retry contract one-sided: the swap is enqueued a
2968
+ // durable local write that cannot fail on the tracker before
2969
+ // `recoveredAt` is written, so the row can never again be taken out of
2970
+ // `runsNeedingClassification` with its label swap still owed. That was
2971
+ // the defect 0.4.4 claimed to have fixed and did not, for this one
2972
+ // recovery; the projector retries until the tracker takes the swap, and
2973
+ // while it is pending the eligibility overlay keeps the issue coherent
2974
+ // (#201).
2975
+ swapToQueue(d, run.issue, inProgress);
2869
2976
  store.updateRun(run.id, {
2870
2977
  state: "killed",
2871
2978
  lastError:
@@ -2906,7 +3013,7 @@ async function recoverRun(
2906
3013
  return;
2907
3014
  }
2908
3015
  const label = cls === "orphan-clean" ? inProgress : project.stateLabels.failed;
2909
- if (!(await swapToQueue(d, run.issue, label))) return;
3016
+ swapToQueue(d, run.issue, label);
2910
3017
  store.updateRun(run.id, { recoveredAt: Date.now() });
2911
3018
  log(`#${run.issue} requeued from ${cls}: ${evidence}`);
2912
3019
  return;
@@ -2972,26 +3079,22 @@ async function recoverRun(
2972
3079
  }
2973
3080
 
2974
3081
  /**
2975
- * Swap a state label for the queue label, in that order.
3082
+ * Enqueue a state-label queue-label swap for projection (#201).
2976
3083
  *
2977
- * Both writes go through the same Tracker port the dispatcher claimed with, so
2978
- * orphan detection stays trustworthy. Returns false when either half failed, so
2979
- * the caller leaves the row for the next tick rather than recording a recovery
2980
- * that did not happen.
3084
+ * The swap is two ops in id order remove first, then add — which is the
3085
+ * atomicity the projector guarantees: the issue never sits newly eligible
3086
+ * without a queue label on its way back, and the add never lands before the
3087
+ * remove when GitHub fails between them. Enqueueing is a durable local write
3088
+ * that cannot fail on the tracker, so the caller records its recovery
3089
+ * immediately and the projector retries the swap until the tracker takes it —
3090
+ * that closes the 0.4.4 hole where a refused label swap stranded the row
3091
+ * permanently under a log line promising a retry.
2981
3092
  */
2982
- async function swapToQueue(
2983
- d: Pick<Deps, "project" | "tracker">,
2984
- issue: number,
2985
- label: string,
2986
- ): Promise<boolean> {
2987
- try {
2988
- await d.tracker.removeLabel(issue, label);
2989
- await d.tracker.addLabel(issue, d.project.queueLabel);
2990
- return true;
2991
- } catch (err) {
2992
- log(`#${issue} could not be requeued (${errText(err)}) — retrying next tick`);
2993
- return false;
2994
- }
3093
+ function swapToQueue(d: Pick<Deps, "project" | "store">, issue: number, label: string): void {
3094
+ d.store.enqueueLabelOps(d.project.name, [
3095
+ { issue, op: "remove", label },
3096
+ { issue, op: "add", label: d.project.queueLabel },
3097
+ ]);
2995
3098
  }
2996
3099
 
2997
3100
  /** Bounded listing per state label, so one reconcile cannot walk a whole repo. */
@@ -3021,13 +3124,11 @@ export async function reconcileStaleLabels(d: Deps): Promise<void> {
3021
3124
  for (const issue of carrying) {
3022
3125
  if (issue.state === "closed") {
3023
3126
  // Never retain an `agent:*` label on a closed issue: the work is done by
3024
- // some route, and the label only makes the board lie about it.
3025
- try {
3026
- await tracker.removeLabel(issue.number, label);
3027
- log(`#${issue.number} reconciled: closed issue no longer carries ${label}`);
3028
- } catch (err) {
3029
- log(`#${issue.number} could not drop ${label} (${errText(err)}) — retrying next tick`);
3030
- }
3127
+ // some route, and the label only makes the board lie about it. Enqueue
3128
+ // rather than call — a refused write must not lose the decision; the
3129
+ // projector retries the removal until the tracker takes it (#201).
3130
+ store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
3131
+ log(`#${issue.number} reconciled: closed issue no longer carries ${label} (queued)`);
3031
3132
  continue;
3032
3133
  }
3033
3134
 
@@ -3038,8 +3139,8 @@ export async function reconcileStaleLabels(d: Deps): Promise<void> {
3038
3139
  const key = `${project.name}:superseded:${issue.number}`;
3039
3140
  if (store.wasNotified(key)) continue;
3040
3141
  const list = children.map((c) => `#${c.number}`).join(", ");
3142
+ store.enqueueLabelOps(project.name, [{ issue: issue.number, op: "remove", label }]);
3041
3143
  try {
3042
- await tracker.removeLabel(issue.number, label);
3043
3144
  await tracker.comment(
3044
3145
  issue.number,
3045
3146
  `superseded: all sub-issues closed (${list}) — this label was stale; propose closing if the ` +
@@ -3048,7 +3149,9 @@ export async function reconcileStaleLabels(d: Deps): Promise<void> {
3048
3149
  store.markNotified(key);
3049
3150
  log(`#${issue.number} reconciled: superseded by ${list}`);
3050
3151
  } catch (err) {
3051
- log(`#${issue.number} could not be reconciled (${errText(err)}) retrying next tick`);
3152
+ // The label removal is already queued and will land regardless; the
3153
+ // comment is the only half that can fail here (#201).
3154
+ log(`#${issue.number} could not comment the superseded note (${errText(err)}) — retrying next tick`);
3052
3155
  }
3053
3156
  }
3054
3157
  }
@@ -3124,7 +3227,19 @@ export async function runDaemon(o: DaemonOpts = {}): Promise<void> {
3124
3227
  const project = findProject(cfg, o.project);
3125
3228
  const caps = resolveCaps(project, cfg.defaults);
3126
3229
  const store = openStore(dbPath());
3127
- const tracker = makeTracker(project);
3230
+ // The tracker's single `gh` funnel is bound to the store so the operator sees
3231
+ // observed truth (#198): every spawn is counted per UTC day, and every
3232
+ // rate-limit refusal is recorded for `status`'s 5m window. Board's ad-hoc
3233
+ // trackers and the polled `fetchRateLimit` probe are deliberately not bound —
3234
+ // this row is the daemon's own traffic.
3235
+ const tracker = makeTracker(project, undefined, {
3236
+ onCall: () => store.bumpGhCalls?.(utcDay(), "daemon"),
3237
+ onRefusal: (at) => store.recordGhRefusal?.(at),
3238
+ // A conditional 304 revalidation is a spawn but not a billed read (#203);
3239
+ // counted separately so `status`'s call row keeps telling the truth once
3240
+ // most spawns are free revalidations.
3241
+ onNotModified: () => store.bumpGhCalls?.(utcDay(), "daemon-304"),
3242
+ });
3128
3243
 
3129
3244
  // #126's transport, stated at startup rather than guessed at first use. The
3130
3245
  // banner names what this host can actually enforce — whether the kernel will