omp-conductor 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -510,7 +510,17 @@ export function failedRunIds(
510
510
  return ids;
511
511
  }
512
512
 
513
- /** Parse `[{number, state}]` from `gh issue list` or the sub-issues API. */
513
+ /**
514
+ * Parse `[{number, state}]` from `gh issue list` or the sub-issues API.
515
+ *
516
+ * The REST issues endpoint mixes pull requests into the list, and the reconcile
517
+ * must never remove labels off a PR — so rows carrying a `pull_request` key are
518
+ * dropped before the caller sees them. This replaces the `--jq
519
+ * select(.pull_request == null)` projection listLabeled used to send: `-i`
520
+ * output (needed for conditional revalidation) cannot carry a jq projection,
521
+ * so the filter moved in-process where it also covers every future caller.
522
+ * `childrenOf` rows never carry the key, so the guard is a no-op there.
523
+ */
514
524
  export function labeledIssuesFrom(raw: string): { number: number; state: IssueState }[] {
515
525
  let parsed: unknown;
516
526
  try {
@@ -523,6 +533,7 @@ export function labeledIssuesFrom(raw: string): { number: number; state: IssueSt
523
533
  for (const entry of parsed) {
524
534
  if (entry === null || typeof entry !== "object") continue;
525
535
  const row = entry as { readonly [key: string]: unknown };
536
+ if (row["pull_request"] != null) continue;
526
537
  const number = row["number"];
527
538
  const rawState = row["state"];
528
539
  if (typeof number !== "number" || !Number.isInteger(number) || typeof rawState !== "string") continue;
@@ -586,6 +597,118 @@ export function readyIssuesFromRest(raw: string): ReadyIssue[] {
586
597
  return issues;
587
598
  }
588
599
 
600
+ /** One cached page per request URL: the ETag GitHub answered, the raw JSON
601
+ * body it covered, and whether that page advertised a rel="next" successor.
602
+ * Module-level default so every makeTracker() in the process (board probe,
603
+ * tick, preview) shares it; keyed by exact URL, so repos, labels and page
604
+ * numbers never collide. */
605
+ export type RestListCache = Map<string, { etag: string; body: string; hasNext: boolean }>;
606
+ const sharedRestListCache: RestListCache = new Map();
607
+
608
+ /**
609
+ * Split a `gh api -i` answer into status, etag, rel="next" presence, body.
610
+ *
611
+ * `gh api -i` prints the status line and response headers, a blank line, then
612
+ * the body. A page's ETag is its identity for conditional revalidation, and
613
+ * the `Link` header is the only place the adapter can learn a successor page
614
+ * exists without parsing the body. Throws on a payload that is not an HTTP
615
+ * response at all — the caller must not guess what an unrecognisable answer
616
+ * means.
617
+ */
618
+ export function splitApiInclude(raw: string): {
619
+ status: number;
620
+ etag?: string;
621
+ hasNext: boolean;
622
+ body: string;
623
+ } {
624
+ const blank = /\r?\n\r?\n/.exec(raw);
625
+ const headerBlock = blank === null ? raw : raw.slice(0, blank.index);
626
+ const body = blank === null ? "" : raw.slice(blank.index + blank[0].length);
627
+ const [statusLine = ""] = headerBlock.split(/\r?\n/);
628
+ const statusMatch = /^HTTP\/[\d.]+ (\d{3})/.exec(statusLine);
629
+ if (statusMatch === null) {
630
+ throw new Error("unrecognised gh api -i response");
631
+ }
632
+ let etag: string | undefined;
633
+ let hasNext = false;
634
+ for (const line of headerBlock.split(/\r?\n/)) {
635
+ const colon = line.indexOf(":");
636
+ if (colon <= 0) continue;
637
+ const name = line.slice(0, colon).trim().toLowerCase();
638
+ const value = line.slice(colon + 1).trim();
639
+ if (name === "etag") etag = value;
640
+ else if (name === "link" && /rel="next"/.test(value)) hasNext = true;
641
+ }
642
+ return { status: Number(statusMatch[1]), etag, hasNext, body };
643
+ }
644
+
645
+ /**
646
+ * GET a REST list path page by page, revalidating each known page with
647
+ * If-None-Match. A 304 reuses that page's cached body and costs no primary
648
+ * rate-limit budget; a changed page answers 200 and replaces its cache entry.
649
+ * Every known page is requested on every pass, so a change on page N is
650
+ * detected even when earlier pages are unchanged. Follows rel="next" with no
651
+ * page cap when `paginate`; entries past the final page are pruned. Returns
652
+ * one raw JSON-array body per page.
653
+ */
654
+ async function conditionalListPages(
655
+ runGh: typeof gh,
656
+ cache: RestListCache,
657
+ path: string,
658
+ paginate: boolean,
659
+ onNotModified?: () => void,
660
+ ): Promise<string[]> {
661
+ const bodies: string[] = [];
662
+ let n = 1;
663
+ for (;;) {
664
+ const url = n === 1 ? path : `${path}&page=${n}`;
665
+ const cached = cache.get(url);
666
+ const argv = ["api", "-i", ...(cached ? ["-H", `If-None-Match: ${cached.etag}`] : []), url];
667
+ let raw: string;
668
+ try {
669
+ raw = await runGh(argv);
670
+ } catch (err) {
671
+ if (!(err instanceof GhError) || !/^HTTP\/[\d.]+ 304\b/.test(err.stdout)) throw err;
672
+ // A 304 is gh's documented answer for an unchanged conditional request:
673
+ // exit 1, with the status line still on stdout. `cached` must exist —
674
+ // we only send If-None-Match when we hold an etag — so rethrow if not,
675
+ // fail-loud, never serve a page we did not cache.
676
+ if (cached === undefined) throw err;
677
+ bodies.push(cached.body);
678
+ onNotModified?.();
679
+ if (!paginate || !cached.hasNext) break;
680
+ n += 1;
681
+ continue;
682
+ }
683
+ const r = splitApiInclude(raw);
684
+ if (r.status === 304) {
685
+ // A future `gh` that exits 0 on a 304 would land here instead of the
686
+ // catch above; both arms answer identically. The cache entry keeps its
687
+ // stored etag — the 304's echo comes back strong-form, while the stored
688
+ // weak form keeps matching.
689
+ if (cached === undefined) {
690
+ throw new Error("a 304 answered a page never sent If-None-Match for");
691
+ }
692
+ bodies.push(cached.body);
693
+ onNotModified?.();
694
+ if (!paginate || !cached.hasNext) break;
695
+ n += 1;
696
+ continue;
697
+ }
698
+ if (r.etag !== undefined) cache.set(url, { etag: r.etag, body: r.body, hasNext: r.hasNext });
699
+ else cache.delete(url);
700
+ bodies.push(r.body);
701
+ if (!paginate || !r.hasNext) break;
702
+ n += 1;
703
+ }
704
+ // The list shrank: without this a later pass would revalidate pages that no
705
+ // longer exist. `n` is the last page fetched; prune every successor key.
706
+ for (let k = n + 1; cache.has(`${path}&page=${k}`); k += 1) {
707
+ cache.delete(`${path}&page=${k}`);
708
+ }
709
+ return bodies;
710
+ }
711
+
589
712
  /** The two GitHub API budgets this adapter spends: GraphQL for the
590
713
  * closers/parent/rollup reads, core for everything else. `reset` is the REST
591
714
  * spelling — epoch seconds. */
@@ -635,8 +758,115 @@ export async function fetchRateLimit(
635
758
  }
636
759
  }
637
760
 
638
- export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
761
+ /**
762
+ * True when a `gh` failure is GitHub refusing us at the rate-limit layer.
763
+ *
764
+ * Three distinct bodies arrive through the CLI and all mean "stop sending":
765
+ * the primary budget exhausted (`API rate limit exceeded`, or the 403 the REST
766
+ * layer reports as `HTTP 403 ... rate limit`), the secondary budget
767
+ * (`secondary rate limit`), and an abuse-control trip (`abuse detection`).
768
+ * gh's stderr carries none of the header values (`Retry-After`, `reset`) that
769
+ * would drive a precise backoff, so the cooldown is the tracker's fixed window
770
+ * below rather than a header-driven one.
771
+ */
772
+ export function isRateLimitRefusal(err: unknown): boolean {
773
+ return (
774
+ err instanceof GhError &&
775
+ /API rate limit exceeded|secondary rate limit|abuse detection|HTTP 403.*rate limit/i.test(
776
+ err.stderr,
777
+ )
778
+ );
779
+ }
780
+
781
+ /**
782
+ * A rate-limit refusal rethrown by {@link makeTracker}. This is the one
783
+ * failure class the tracker names instead of swallowing: the circuit breaker
784
+ * throws it without spawning `gh`, and it is what a caller can distinguish
785
+ * from an ordinary transport fault.
786
+ */
787
+ export class GhRateLimitError extends GhError {
788
+ /** Wall-clock millis after which a call may be attempted again. */
789
+ readonly retryAtMs: number;
790
+
791
+ constructor(argv: string[], code: number, stderr: string, retryAtMs: number, stdout = "") {
792
+ super(argv, code, stderr, stdout);
793
+ this.name = "GhRateLimitError";
794
+ this.retryAtMs = retryAtMs;
795
+ this.message = `${this.message} — GitHub rate limit; retry at ${new Date(retryAtMs).toISOString()}`;
796
+ }
797
+ }
798
+
799
+ /** How long a rate-limit refusal holds the circuit breaker open. */
800
+ export const RATE_LIMIT_COOLDOWN_MS = 60_000;
801
+
802
+ /** Instrumentation hooks the daemon binds the tracker to (#198). */
803
+ export interface TrackerHooks {
804
+ /** Fired immediately before each `gh` spawn, so the daemon can count its own
805
+ * API spend. Not fired for a breaker fast-fail, which spawns nothing. */
806
+ onCall?: () => void;
807
+ /** Fired once per observed refusal, with the wall-clock moment. */
808
+ onRefusal?: (at: number) => void;
809
+ /** Fired once per list page GitHub answered 304 for — a spawn that cost no
810
+ * primary-rate-limit budget. Lets the daemon's call counter tell billed
811
+ * reads from free revalidations (#203 on top of #198's counting). */
812
+ onNotModified?: () => void;
813
+ }
814
+
815
+ interface TrackerOpts {
816
+ /** Override {@link RATE_LIMIT_COOLDOWN_MS} (tests use a 0ms window). */
817
+ rateLimitCooldownMs?: number;
818
+ /** Conditional-request cache for REST list reads; defaults to the shared
819
+ * module-level cache. Tests pass a fresh Map. */
820
+ listCache?: RestListCache;
821
+ }
822
+
823
+ export function makeTracker(
824
+ p: ProjectConfig,
825
+ injectedRunGh: typeof gh = gh,
826
+ hooks: TrackerHooks = {},
827
+ opts: TrackerOpts = {},
828
+ ): Tracker {
639
829
  const repo = p.tracker.repo;
830
+ const refuseForMs = opts.rateLimitCooldownMs ?? RATE_LIMIT_COOLDOWN_MS;
831
+ const cache = opts.listCache ?? sharedRestListCache;
832
+
833
+ // The single funnel around the injected `gh`. A caught refusal records the
834
+ // user-observed moment, opens the 60s breaker and rethrows named; while the
835
+ // breaker is open every call fails fast WITHOUT spawning gh — a burst under a
836
+ // persistent limit degrades the tick into holds (`open-pr-lookup-error` /
837
+ // `parent-lookup-error` fail closed per candidate and retry next tick)
838
+ // instead of an error storm or a stalled loop. No in-wrapper sleeps.
839
+ let refusedUntil = 0;
840
+ const runGh = async (argv: string[], stdin?: string): Promise<string> => {
841
+ const now = Date.now();
842
+ if (now < refusedUntil) {
843
+ throw new GhRateLimitError(
844
+ argv,
845
+ 0,
846
+ "circuit breaker open (a prior GitHub rate-limit refusal holds the cooldown)",
847
+ refusedUntil,
848
+ );
849
+ }
850
+ hooks.onCall?.();
851
+ try {
852
+ return await injectedRunGh(argv, stdin);
853
+ } catch (err) {
854
+ if (isRateLimitRefusal(err)) {
855
+ const at = Date.now();
856
+ refusedUntil = at + refuseForMs;
857
+ hooks.onRefusal?.(at);
858
+ const cause = err instanceof GhError ? err : undefined;
859
+ throw new GhRateLimitError(
860
+ argv,
861
+ cause?.code ?? 1,
862
+ cause?.stderr ?? String(err),
863
+ refusedUntil,
864
+ cause?.stdout ?? "",
865
+ );
866
+ }
867
+ throw err;
868
+ }
869
+ };
640
870
 
641
871
  // Named rather than returned inline, so `rerunFailedChecks` can reuse
642
872
  // `checkConclusions` instead of re-implementing the same `gh` call.
@@ -644,18 +874,33 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
644
874
  async listReady(): Promise<ReadyIssue[]> {
645
875
  // REST, not `issue list --json`: the ready-queue read is a hot-path poll
646
876
  // on every dispatch pass, and GraphQL budget is the scarce one (#188).
647
- // ponytail: one page is the cap. A queue deeper than 100 ready issues
648
- // truncates silently; upgrade path is `--paginate`, but a backlog that
649
- // size is a staffing problem before it is a paging one.
650
- const raw = await runGh([
651
- "api",
877
+ // Paginated rather than one page of 100 now that an idle revalidation
878
+ // costs an unconditional-200's worth of nothing, truncating a deep queue
879
+ // to save calls saves nothing that matters; the cap was the call count,
880
+ // and the conditional path changed what a call costs.
881
+ const pages = await conditionalListPages(
882
+ runGh,
883
+ cache,
652
884
  `repos/${repo}/issues?state=open&labels=${encodeURIComponent(p.queueLabel)}&per_page=100`,
653
- ]);
885
+ /* paginate */ true,
886
+ hooks.onNotModified,
887
+ );
654
888
 
655
889
  // The REST endpoint always answers a JSON array — an empty queue is `[]`,
656
890
  // not empty text — but an empty body still parses to no issues here, and
657
891
  // the empty queue is the normal steady state, not an error.
658
- return readyIssuesFromRest(raw);
892
+ return pages.flatMap(readyIssuesFromRest);
893
+ },
894
+
895
+ async listOpenIssues(): Promise<ReadyIssue[]> {
896
+ const pages = await conditionalListPages(
897
+ runGh,
898
+ cache,
899
+ `repos/${repo}/issues?state=open&per_page=100`,
900
+ /* paginate */ true,
901
+ hooks.onNotModified,
902
+ );
903
+ return pages.flatMap(readyIssuesFromRest);
659
904
  },
660
905
 
661
906
  async addLabel(issue: number, label: string): Promise<void> {
@@ -902,16 +1147,19 @@ export function makeTracker(p: ProjectConfig, runGh: typeof gh = gh): Tracker {
902
1147
 
903
1148
  async listLabeled(label: string, limit = 50): Promise<{ number: number; state: IssueState }[]> {
904
1149
  try {
905
- return labeledIssuesFrom(
906
- await runGh([
907
- "api",
908
- `repos/${repo}/issues?state=all&labels=${encodeURIComponent(label)}&per_page=${limit}`,
909
- "--jq",
910
- // The endpoint mixes pull requests into the issue list; the
911
- // reconcile must not remove labels off PRs.
912
- "[.[] | select(.pull_request == null) | {number, state}]",
913
- ]),
1150
+ // One page, deliberately: `limit` is the caller's work bound
1151
+ // (RECONCILE_LIMIT), not a queue-depth cap, and the reconcile only
1152
+ // ever acts on the page it reads. The conditional cache still makes a
1153
+ // steady-state pass cost nothing even though the response is big.
1154
+ const pages = await conditionalListPages(
1155
+ runGh,
1156
+ cache,
1157
+ `repos/${repo}/issues?state=all&labels=${encodeURIComponent(label)}&per_page=${limit}`,
1158
+ /* paginate */ false,
1159
+ hooks.onNotModified,
914
1160
  );
1161
+ // paginate: false means exactly one body, always a JSON array.
1162
+ return labeledIssuesFrom(pages[0] ?? "");
915
1163
  } catch {
916
1164
  // A reconcile that cannot list must remove no labels: an empty answer is
917
1165
  // read as "no evidence", never as "nothing carries this label".
package/src/types.ts CHANGED
@@ -694,6 +694,14 @@ export interface OpenCloser {
694
694
  */
695
695
  export interface Tracker {
696
696
  listReady(): Promise<ReadyIssue[]>;
697
+ /**
698
+ * Every open issue in the tracker repo (never PRs), labels included: one
699
+ * read that covers all lifecycle-label queries of a pass, so consumers
700
+ * needing several label sets pay one fetch instead of one per label, and
701
+ * openness itself is answerable without a per-issue lookup (#203).
702
+ * Complete — follows pagination to the end.
703
+ */
704
+ listOpenIssues(): Promise<ReadyIssue[]>;
697
705
  addLabel(issue: number, label: string): Promise<void>;
698
706
  removeLabel(issue: number, label: string): Promise<void>;
699
707
  comment(issue: number, body: string): Promise<void>;
@@ -911,6 +919,11 @@ export interface RunRecord {
911
919
  * exactly as an unflagged one does, and the flags are evidence for whoever
912
920
  * reviews the PR. Absent means the audit found nothing, or never ran. */
913
921
  settlementFlags?: SettlementFlag[];
922
+ /** The worker's settlement report text for this attempt, verbatim. Persisted
923
+ * on every terminal update so the next attempt can pool its disclosures
924
+ * (#199). Absent means the run predates the column, or was killed before the
925
+ * worker returned a report. */
926
+ report?: string;
914
927
  /**
915
928
  * Why this run ended badly, and what the daemon did about it (#132). Absent
916
929
  * means the sweep has not looked at the row yet — never "nothing was wrong":
@@ -1143,10 +1156,28 @@ export interface DecisionDraft {
1143
1156
  at: number;
1144
1157
  }
1145
1158
 
1159
+ /** One decided GitHub label change waiting to be projected (#201). */
1160
+ export interface LabelOp {
1161
+ id: number;
1162
+ project: string;
1163
+ issue: number;
1164
+ op: "add" | "remove";
1165
+ label: string;
1166
+ createdAt: number;
1167
+ attempts: number;
1168
+ nextAttemptAt: number;
1169
+ lastError?: string;
1170
+ }
1171
+
1146
1172
  /**
1147
- * Bookkeeping only GitHub labels remain the source of truth. The store
1148
- * exists to answer cap questions cheaply and to survive a restart; if it is
1149
- * ever lost, the tracker can rebuild the world.
1173
+ * Dispatch state is store-authoritative; GitHub labels are a retried
1174
+ * write-behind projection of it (#201). `isEligible` still consults the label
1175
+ * set a candidate *effectively* carries physical labels overlaid with the
1176
+ * outbox's pending ops — so a label GitHub refuses to move cannot strand an
1177
+ * issue the daemon has already decided to move on. A store loss no longer
1178
+ * rebuilds the world: it also forgets what labels the tracker was told to
1179
+ * carry, so the operator reconciles by hand, which is the only safe move once
1180
+ * the two disagree.
1150
1181
  */
1151
1182
  export interface Store {
1152
1183
  createRun(r: Omit<RunRecord, "id">): RunRecord;
@@ -1179,6 +1210,11 @@ export interface Store {
1179
1210
  * tail` resolves an issue number to a transcript through this; the number is
1180
1211
  * what an operator has, the run id is not. */
1181
1212
  latestRun(project: string, issue: number): RunRecord | undefined;
1213
+ /** Every attempt of one issue that settled with a report, in attempt order.
1214
+ * The settlement audit pools these as prior disclosures when a later
1215
+ * attempt is reconciled (#199). Rows with no report (pre-#199, or killed
1216
+ * before the worker returned) are skipped. */
1217
+ attemptReports(project: string, issue: number): { attempt: number; report: string }[];
1182
1218
  runsStartedSince(project: string, sinceEpochMs: number): number;
1183
1219
  spendSince(project: string, sinceEpochMs: number): number;
1184
1220
  /** Idempotence guard so a retry loop cannot page a human repeatedly for the
@@ -1198,6 +1234,15 @@ export interface Store {
1198
1234
  /** Start the cooldown only after a tick carrying these signals was sent. */
1199
1235
  markFrictionSurfaced(project: string, kinds: readonly FrictionKind[], at: number): void;
1200
1236
  markNotified(key: string): void;
1237
+ /** Record one observed GitHub rate-limit refusal (the tracker's hook). Rows
1238
+ * older than 24h are pruned in the same write (#198). */
1239
+ recordGhRefusal?(at: number): void;
1240
+ /** Refusals observed at or after `sinceMs` — `status` reads the last 5m. */
1241
+ ghRefusalsSince?(sinceMs: number): { count: number; latestAt?: number };
1242
+ /** Count one daemon-held `gh` call for the UTC day (`YYYY-MM-DD`). */
1243
+ bumpGhCalls?(day: string, source: string): void;
1244
+ /** The tracked call counts for a UTC day, per source. */
1245
+ ghCallsToday?(day: string): { source: string; calls: number }[];
1201
1246
  /**
1202
1247
  * Persist a rendered report `pending`, before anything is sent — the whole
1203
1248
  * point of #123 is that an undelivered report is a queryable row rather than
@@ -1292,6 +1337,33 @@ export interface Store {
1292
1337
  releaseMergeLock(project: string, holder: string): void;
1293
1338
  /** The live lock, for `status` and for tests. */
1294
1339
  mergeLock(project: string): MergeLock | undefined;
1340
+ /**
1341
+ * Append decided GitHub label changes to the projection outbox (#201). One
1342
+ * transaction: either every op lands as a pending row or none do, so the
1343
+ * daemon's record of its own intent cannot disagree with itself. Ops for one
1344
+ * issue apply in enqueue (id) order, which is what makes a swap atomic — the
1345
+ * remove lands before the add exactly as enqueued.
1346
+ */
1347
+ enqueueLabelOps(
1348
+ project: string,
1349
+ ops: readonly { issue: number; op: "add" | "remove"; label: string }[],
1350
+ ): void;
1351
+ /** Unapplied ops whose retry backoff has elapsed, oldest first (#201). */
1352
+ pendingLabelOps(project: string, now: number): LabelOp[];
1353
+ /** Every unapplied op for one issue, enqueue order — the eligibility overlay
1354
+ * reads this so a stale GitHub label cannot outlive the daemon's intent
1355
+ * (#201). */
1356
+ pendingLabelOpsFor(project: string, issue: number): LabelOp[];
1357
+ /** The op applied; forget it. */
1358
+ settleLabelOp(id: number): void;
1359
+ /** The op failed; count the attempt and park it behind `nextAttemptAt`. Pass
1360
+ * `countAttempt: false` for a refusal that is not the op's fault (a shared
1361
+ * rate limit), so it does not burn the backoff escalation (#208). */
1362
+ deferLabelOp(id: number, error: string, nextAttemptAt: number, countAttempt?: boolean): void;
1363
+ /** Unapplied ops still owed — `status` shows the projection lag (#201). */
1364
+ countPendingLabelOps(project: string): number;
1365
+ /** `createdAt` of the oldest unapplied op, or `undefined` for none. */
1366
+ oldestPendingLabelOpAt(project: string): number | undefined;
1295
1367
  close(): void;
1296
1368
  }
1297
1369
 
package/src/unblock.ts CHANGED
@@ -20,16 +20,22 @@
20
20
  * dispatchable again, and it stays off under `--no-requeue` or while any run
21
21
  * for the issue is still active.
22
22
  *
23
- * Nothing here writes to the store, and that is a decision rather than an
24
- * omission. `RunState` describes what a worker process did; an answer is the
23
+ * Nothing here writes to the run history, and that is a decision rather than
24
+ * an omission. `RunState` describes what a worker process did; an answer is the
25
25
  * one event that happens outside every run, so no member fits it — folding it
26
26
  * into `merged` or `killed` would make `status` describe a run that never
27
- * reached either. Eligibility is read off the tracker's labels and never off a
28
- * run row, so the store has nothing to say here. Leaving history alone keeps
29
- * both budgets honest: a block consumes an operational continuation, while a
30
- * real implementation failure consumes the separate failed-attempt budget.
27
+ * reached either. The one store write is the label projection outbox (#201):
28
+ * a decided label change is durable locally and projected with retry, so a 403
29
+ * defers it instead of throwing the verb, and the daemon's next tick applies it
30
+ * even if this process dies before it does. Eligibility is read off the label
31
+ * set an issue effectively carries — physical labels overlaid with that outbox —
32
+ * and never off a run row, so the store has nothing else to say here. Leaving
33
+ * history alone keeps both budgets honest: a block consumes an operational
34
+ * continuation, while a real implementation failure consumes the separate
35
+ * failed-attempt budget.
31
36
  */
32
37
 
38
+ import { projectLabels } from "./label-projection.ts";
33
39
  import { LIVE_STATES } from "./store.ts";
34
40
  import type { Caps, ProjectConfig, RunRecord, Store, Tracker } from "./types.ts";
35
41
 
@@ -65,6 +71,14 @@ export interface UnblockOutcome {
65
71
  requeued?: true;
66
72
  /** Set when `--no-requeue` skipped the queue-label re-add. */
67
73
  requeueSkipped?: true;
74
+ /** Set when the tracker refused one or more of this verb's label ops, so
75
+ * the label sync is owed rather than done: the intended label state is
76
+ * durable in the store and the daemon retries it (#201). Safety is
77
+ * preserved either way, but the issue is only claimable once the queue
78
+ * label itself has landed — `listReady` reads the tracker by label, so a
79
+ * queue-label add that has not been applied is invisible to dispatch. The
80
+ * number is what is still pending. */
81
+ labelSyncQueued?: number;
68
82
  }
69
83
 
70
84
  /**
@@ -150,12 +164,13 @@ export async function unblockIssue(
150
164
  if (held !== undefined) store.updateRun(held.id, { salvageAckAt: Date.now() });
151
165
 
152
166
  const cleared: string[] = [];
167
+ const ops: { issue: number; op: "add" | "remove"; label: string }[] = [];
153
168
  for (const label of new Set([
154
169
  project.stateLabels.blocked,
155
170
  project.stateLabels.failed,
156
171
  ...(terminal ? [project.stateLabels.inProgress] : []),
157
172
  ])) {
158
- await tracker.removeLabel(issue, label);
173
+ ops.push({ issue, op: "remove", label });
159
174
  cleared.push(label);
160
175
  }
161
176
 
@@ -173,15 +188,33 @@ export async function unblockIssue(
173
188
  // the PR resolves closed-unmerged.
174
189
  let requeued: true | undefined;
175
190
  let requeueSkipped: true | undefined;
191
+ let labelSyncQueued: number | undefined;
176
192
  if (!live) {
177
193
  if (requeue) {
178
- await tracker.addLabel(issue, project.queueLabel);
194
+ ops.push({ issue, op: "add", label: project.queueLabel });
179
195
  requeued = true;
180
196
  } else {
181
197
  requeueSkipped = true;
182
198
  }
183
199
  }
184
200
 
201
+ // Apply through the same projector the daemon drains (#201): the clears and
202
+ // the queue restore land in enqueue order, and a tracker that refuses (403,
203
+ // rate limit) defers them with backoff instead of throwing the verb — which
204
+ // is the concrete #184/#198 kill: `unblock` stops breaking on a 403. The
205
+ // intended label state is durable; the daemon converges the tracker. The
206
+ // issue only becomes claimable once the queue label itself lands —
207
+ // `listReady` queries by label, so a pending queue-label add is invisible to
208
+ // dispatch until projection succeeds.
209
+ if (ops.length > 0) {
210
+ store.enqueueLabelOps(project.name, ops);
211
+ await projectLabels(store, tracker, project);
212
+ // How much of THIS issue's sync is still owed (not the whole fleet's outbox):
213
+ // a backlog on other issues is not this verb's lag to report.
214
+ const stillPending = store.pendingLabelOpsFor(project.name, issue).length;
215
+ if (stillPending > 0) labelSyncQueued = stillPending;
216
+ }
217
+
185
218
  return {
186
219
  cleared,
187
220
  ...counts,
@@ -190,6 +223,7 @@ export async function unblockIssue(
190
223
  ...(held === undefined ? {} : { forced: true as const }),
191
224
  ...(requeued === undefined ? {} : { requeued }),
192
225
  ...(requeueSkipped === undefined ? {} : { requeueSkipped }),
226
+ ...(labelSyncQueued === undefined ? {} : { labelSyncQueued }),
193
227
  };
194
228
  }
195
229
 
@@ -228,6 +262,15 @@ export function formatUnblock(
228
262
  }
229
263
 
230
264
  const lines = [`#${issue}: cleared ${o.cleared.join(", ")}`];
265
+ // When the tracker refused the sync, NO label is provably applied — the
266
+ // "restored / eligible again" claims below would be a lie, so they branch to
267
+ // the queued/retry wording instead (#201).
268
+ const labelsPending = o.labelSyncQueued !== undefined;
269
+ if (o.labelSyncQueued !== undefined) {
270
+ lines.push(
271
+ ` label sync queued (${o.labelSyncQueued} pending) — the daemon retries; the intended labels are durable, and the issue is claimable once they land`,
272
+ );
273
+ }
231
274
  if (o.forced === true) {
232
275
  lines.push(
233
276
  ` forced the unsalvaged worktree was accepted as lost or already recovered by hand — ` +
@@ -277,11 +320,13 @@ export function formatUnblock(
277
320
  // instead of promising a re-claim the next tick withholds.
278
321
  lines.push(
279
322
  latest.prUrl === undefined
280
- ? o.requeued === true
281
- ? ` next tick eligible again "${project.queueLabel}" restored (no-op if it was already present; ` +
282
- "the dispatcher still applies its open-PR check at claim time)"
283
- : ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
284
- `(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
323
+ ? labelsPending
324
+ ? ` next tick label sync queued (${o.labelSyncQueued} pending) the daemon retries; eligibility is recorded in the store, not yet on the tracker`
325
+ : o.requeued === true
326
+ ? ` next tick eligible again "${project.queueLabel}" restored (no-op if it was already present; ` +
327
+ "the dispatcher still applies its open-PR check at claim time)"
328
+ : ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
329
+ `(newest run is ${latest.state} with no recorded PR; the dispatcher still applies its open-PR check at claim time)`
285
330
  : ` next tick eligible as a continuation of ${latest.prUrl} — the pushed run stays active until ` +
286
331
  "that PR resolves; dispatch continues on its branch",
287
332
  );
@@ -305,17 +350,23 @@ export function formatUnblock(
305
350
  ` next tick no live worker is on this issue — its active run(s) are worker-free pushes; ` +
306
351
  "dispatch continues their branch once its holds and open-PR check clear",
307
352
  );
308
- if (o.requeued === true) {
353
+ if (labelsPending) {
354
+ lines.push(
355
+ ` queue label sync queued (${o.labelSyncQueued} pending) — the daemon retries; the intended labels are durable, and the issue is claimable once they land`,
356
+ );
357
+ } else if (o.requeued === true) {
309
358
  lines.push(` queue "${project.queueLabel}" restored (no-op if it was already present)`);
310
359
  }
311
360
  }
312
361
  } else {
313
362
  lines.push(
314
- o.requeued === true
315
- ? ` next tick eligible again "${project.queueLabel}" restored (no-op if it was already present; ` +
316
- "the dispatcher still applies its open-PR check at claim time)"
317
- : ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
318
- "(the dispatcher still applies its open-PR check at claim time)",
363
+ labelsPending
364
+ ? ` next tick label sync queued (${o.labelSyncQueued} pending) the daemon retries; the intended labels are durable, and the issue is claimable once they land`
365
+ : o.requeued === true
366
+ ? ` next tick eligible again "${project.queueLabel}" restored (no-op if it was already present; ` +
367
+ "the dispatcher still applies its open-PR check at claim time)"
368
+ : ` next tick eligible again, as long as the issue still carries "${project.queueLabel}" ` +
369
+ "(the dispatcher still applies its open-PR check at claim time)",
319
370
  );
320
371
  }
321
372