gh-glance 0.15.2 → 0.16.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/CHANGELOG.md +52 -1
  2. package/README.md +76 -25
  3. package/index.mjs +2237 -500
  4. package/package.json +1 -1
package/index.mjs CHANGED
@@ -22,7 +22,7 @@
22
22
  // disarmDevBuildLeak() for how that path is kept survivable rather than fatal.
23
23
  process.env.NODE_ENV ??= "production";
24
24
 
25
- import { execFile, spawn } from "node:child_process";
25
+ import { execFile, execFileSync, spawn } from "node:child_process";
26
26
  import { AsyncLocalStorage } from "node:async_hooks";
27
27
  import { createHash, createHmac, createPrivateKey, randomUUID, sign as cryptoSign, timingSafeEqual } from "node:crypto";
28
28
  import { EventEmitter } from "node:events";
@@ -207,7 +207,7 @@ const STALE_AFTER_MS = 30_000;
207
207
  // A coordination hold must persist long enough to be meaningful before it
208
208
  // takes the notice row. Short startup and lock-contention states recover on
209
209
  // their own and should not present themselves as user-visible failures.
210
- const COORDINATION_NOTICE_AFTER_MS = 2_000;
210
+ const COORDINATION_NOTICE_AFTER_MS = 800;
211
211
 
212
212
  // Mutable because the fetchers below are defined before argv is parsed, and the
213
213
  // argv block is what fills this in. Everything here has a working default, so
@@ -221,7 +221,7 @@ const runtime = {
221
221
  host: null,
222
222
  repoExplicit: false,
223
223
  refreshMs: REFRESH_MS,
224
- background: "all",
224
+ background: "on-demand",
225
225
  verbose: false,
226
226
  connect: null,
227
227
  initialTabIndex: 0,
@@ -734,7 +734,9 @@ const pausedOperations = new Map();
734
734
 
735
735
  function pauseOperation(operation, untilMs) {
736
736
  if (!Number.isFinite(untilMs)) return;
737
- pausedOperations.set(operation, Math.max(pausedOperations.get(operation) ?? 0, untilMs));
737
+ const until = Math.max(pausedOperations.get(operation) ?? 0, untilMs);
738
+ pausedOperations.set(operation, until);
739
+ return () => { if (pausedOperations.get(operation) === until) pausedOperations.delete(operation); };
738
740
  }
739
741
 
740
742
  function operationPausedUntil(operation, now = Date.now()) {
@@ -744,7 +746,8 @@ function operationPausedUntil(operation, now = Date.now()) {
744
746
  return until;
745
747
  }
746
748
 
747
- async function runGh(args, { signal, operation, input = null, execute = null } = {}) {
749
+ async function runGh(args, { signal, operation, input = null, execute = null,
750
+ closeWaitMs = 15_000 } = {}) {
748
751
  if (operationCost(operation) === null) {
749
752
  throw new Error(`undeclared gh operation: ${operation ?? "missing"}`);
750
753
  }
@@ -756,11 +759,32 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
756
759
  const requestNow = typeof bound?.now === "function" ? bound.now : Date.now;
757
760
  const permit = identityCoordinator && !local
758
761
  ? await acquireIdentityHttpPermit(identityCoordinator, { signal, now: requestNow,
762
+ operationDeadline: CONTROL_OPERATIONS.includes(operation) ? undefined : bound?.receiptCapability?.deadline,
759
763
  ...(typeof bound?.httpWait === "function" ? { wait: bound.httpWait } : {}) }) : null;
760
764
  const startedAt = requestNow();
761
765
  let requestError = null;
762
766
  let requestStdout = null;
763
767
  let requestStarted = false;
768
+ let knownNeverStarted = false;
769
+ let requestSettled = false;
770
+ let successfulPromise = false;
771
+ let dispatchEvidence = null;
772
+ let dispatchCapability = null;
773
+ let childCloseProof = null;
774
+ let activeDispatchKey = null;
775
+ let activeDispatchReleased = false;
776
+ const releaseActiveDispatch = () => {
777
+ if (!activeDispatchKey || activeDispatchReleased) return;
778
+ activeDispatchReleased = true;
779
+ const remaining = (activeOwnerDispatches.get(activeDispatchKey) ?? 1) - 1;
780
+ if (remaining > 0) activeOwnerDispatches.set(activeDispatchKey, remaining);
781
+ else activeOwnerDispatches.delete(activeDispatchKey);
782
+ if (dispatchCapability && !dispatchCapability.controlResource) {
783
+ const sealed = inspectGovernor({ ...bound, identityProvider: null }, requestNow());
784
+ const nonce = sealed.ok && sealed.value.debtGroups[dispatchCapability.generation]?.nonce;
785
+ if (nonce) acknowledgeSealedGeneration(bound, dispatchCapability, nonce, requestNow());
786
+ }
787
+ };
764
788
  try {
765
789
  assertBoundCredential(bound);
766
790
  if (identityCoordinator && operation === "budget-core-observer") {
@@ -789,6 +813,23 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
789
813
  }
790
814
  }
791
815
  assertBoundCredential(bound);
816
+ if (!local && RATE_RESOURCES.some((resource) => operationCost(operation)[resource] > 0)) {
817
+ dispatchCapability = operation === "budget-core-observer"
818
+ ? control?.value?.receiptCapability
819
+ : operation === "graphql-observer"
820
+ ? bound?.controlReceiptCapability : bound?.receiptCapability;
821
+ if (dispatchCapability) {
822
+ activeDispatchKey = ownerDispatchKey(bound, dispatchCapability);
823
+ activeOwnerDispatches.set(activeDispatchKey,
824
+ (activeOwnerDispatches.get(activeDispatchKey) ?? 0) + 1);
825
+ }
826
+ const issued = issueGovernorDispatch(bound, dispatchCapability, operation, requestNow());
827
+ if (!issued.ok) {
828
+ releaseActiveDispatch();
829
+ throw new Error(`API receipt paused (${issued.reason})`);
830
+ }
831
+ dispatchEvidence = issued.value;
832
+ }
792
833
  const executeChild = execute ?? bound?.executeGh ?? execFileAsync;
793
834
  const pending = executeChild("gh", args, {
794
835
  timeout: GH_TIMEOUT_MS,
@@ -798,6 +839,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
798
839
  signal,
799
840
  });
800
841
  requestStarted = true;
842
+ if (Number.isSafeInteger(pending.child?.pid) &&
843
+ typeof pending.child.once === "function") {
844
+ childCloseProof = new Promise((resolve) => pending.child.once("close", resolve));
845
+ }
846
+ if (dispatchEvidence && Number.isSafeInteger(pending.child?.pid)) {
847
+ markGovernorDispatchChild(bound, dispatchCapability,
848
+ dispatchEvidence.sequence, pending.child.pid, requestNow());
849
+ }
801
850
  if (typeof input === "string") {
802
851
  // A child killed by the timeout or the abort signal closes stdin under
803
852
  // us; that EPIPE is the kill's consequence, not the failure worth
@@ -805,16 +854,73 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
805
854
  pending.child.stdin?.on("error", () => {});
806
855
  pending.child.stdin?.end(input);
807
856
  }
808
- const { stdout } = await pending;
857
+ let stdout;
858
+ try {
859
+ ({ stdout } = await pending);
860
+ requestSettled = true;
861
+ successfulPromise = true;
862
+ } catch (error) {
863
+ requestSettled = true;
864
+ throw error;
865
+ }
809
866
  requestStdout = stdout;
810
- logGh(args, startedAt, `ok ${stdout.length}B`);
811
- return stdout;
812
867
  } catch (err) {
813
868
  requestError = err;
869
+ if (!requestStarted && !dispatchEvidence) err.notStarted ??= true;
870
+ knownNeverStarted = !requestStarted && err?.notStarted === true;
814
871
  if (requestStarted && isRecord(err)) err.httpStarted = true;
815
872
  logGh(args, startedAt, `FAILED ${shortErr(err)}`);
816
873
  throw err;
817
874
  } finally {
875
+ let childClosed = !requestStarted;
876
+ if (requestSettled && childCloseProof) {
877
+ let timer;
878
+ childClosed = await Promise.race([
879
+ childCloseProof.then(() => true),
880
+ new Promise((resolve) => { timer = setTimeout(() => resolve(false), closeWaitMs); }),
881
+ ]);
882
+ clearTimeout(timer);
883
+ }
884
+ if (requestSettled && !childClosed) {
885
+ requestError ??= Object.assign(new Error("GitHub child close unverified"), { httpStarted: true });
886
+ }
887
+ if (successfulPromise) {
888
+ logGh(args, startedAt, childClosed
889
+ ? `ok ${requestStdout.length}B` : "FAILED GitHub child close unverified");
890
+ }
891
+ if (dispatchEvidence && (requestSettled && childClosed || knownNeverStarted)) {
892
+ const finishDispatch = () => {
893
+ const result = terminalGovernorDispatch(bound,
894
+ dispatchCapability, dispatchEvidence.sequence, requestNow(),
895
+ { neverStarted: knownNeverStarted });
896
+ if (result.ok || result.reason === "compacted") releaseActiveDispatch();
897
+ return result;
898
+ };
899
+ const terminal = await retryIdentityCompletion(finishDispatch, {
900
+ timeoutMs: 15_000, shouldRetry: () => true,
901
+ });
902
+ if (!terminal.ok && identityCoordinator?.deferCompletion) {
903
+ identityCoordinator.deferCompletion(
904
+ `dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
905
+ finishDispatch,
906
+ );
907
+ }
908
+ } else if (dispatchEvidence && requestSettled && childCloseProof) {
909
+ // A promise can settle before its subprocess closes. Keep the issued
910
+ // charge until close proves that no more HTTP can escape this child.
911
+ void childCloseProof.then(async () => {
912
+ const finishDispatch = () => terminalGovernorDispatch(bound,
913
+ dispatchCapability, dispatchEvidence.sequence, requestNow());
914
+ const terminal = await retryIdentityCompletion(finishDispatch, {
915
+ timeoutMs: 15_000, shouldRetry: () => true,
916
+ });
917
+ if (terminal.ok || terminal.reason === "compacted") releaseActiveDispatch();
918
+ else identityCoordinator?.deferCompletion?.(
919
+ `dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
920
+ finishDispatch,
921
+ );
922
+ });
923
+ }
818
924
  if (permit) {
819
925
  const release = () => releaseIdentityHttpPermit(identityCoordinator, permit, requestError, requestNow());
820
926
  const released = await retryIdentityCompletion(release, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
@@ -822,11 +928,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
822
928
  }
823
929
  if (control?.ok) {
824
930
  const settle = () => settleIdentityControl(identityCoordinator, control.value,
825
- requestStdout ?? requestError?.stdout, requestNow());
931
+ childClosed ? requestStdout ?? requestError?.stdout : null, requestNow(),
932
+ { neverStarted: !requestStarted && (!dispatchEvidence || knownNeverStarted) });
826
933
  const settled = await retryIdentityCompletion(settle, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
827
934
  if (!settled.ok) identityCoordinator.deferCompletion(`control:${control.value.id}`, settle);
828
935
  }
829
936
  }
937
+ if (successfulPromise && requestError) throw requestError;
938
+ return requestStdout;
830
939
  }
831
940
 
832
941
  function parseGhApiResponse(stdout) {
@@ -1638,7 +1747,6 @@ async function fetchActions(signal, {
1638
1747
  }
1639
1748
  const responses = [runsResponse];
1640
1749
  let requestMetrics = runsResponse.requestMetrics ?? restResponseRequestMetrics(runsResponse);
1641
- const uncertainReceipts = [];
1642
1750
  const runs = parseActionsRuns(runsResponse.body);
1643
1751
  const startedAt = Date.now();
1644
1752
  let nextCatalog = catalog;
@@ -1662,7 +1770,6 @@ async function fetchActions(signal, {
1662
1770
  request,
1663
1771
  }),
1664
1772
  });
1665
- uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
1666
1773
  if (admitted.ok && admitted.value) {
1667
1774
  responses.push(admitted.value);
1668
1775
  requestMetrics = mergeAcquisitionRequestMetrics(
@@ -1695,7 +1802,6 @@ async function fetchActions(signal, {
1695
1802
  failedRequests: requestMetrics.failedRequests ?? 0,
1696
1803
  coreUnitsTotal: requestMetrics.coreUnits ?? 0,
1697
1804
  requestMetrics,
1698
- uncertainReceipts,
1699
1805
  graphqlSpent: GRAPHQL_PER_FETCH.actions,
1700
1806
  stagedEntities: batch.stagedEntities,
1701
1807
  observations: batch.observations,
@@ -1896,7 +2002,6 @@ async function fetchGraphqlList(kind, mapRow, {
1896
2002
  throw error;
1897
2003
  }
1898
2004
  const observations = [...first.observations];
1899
- const uncertainReceipts = [];
1900
2005
  const repositoryIdentity = normalizeRepositoryIdentity(first.data?.repository);
1901
2006
  // Only the first page. Pages past it opened their own reservation inside
1902
2007
  // runAdmittedOperation and settle against it, so adding them here charges the
@@ -1934,7 +2039,6 @@ async function fetchGraphqlList(kind, mapRow, {
1934
2039
  waitMs: GOVERNOR_ADMISSION_WAIT_MS,
1935
2040
  run: (admittedSignal) => fetchPage(kind, { signal: admittedSignal, after: cursor, operation: `page:${kind}` }),
1936
2041
  });
1937
- uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
1938
2042
  // A later page that is denied, fails, or returns errors leaves the rows
1939
2043
  // already gathered exactly as they are. Losing page one because page two
1940
2044
  // was refused would turn a budget decision into data loss.
@@ -1988,7 +2092,6 @@ async function fetchGraphqlList(kind, mapRow, {
1988
2092
  httpRequests: requestMetrics.httpRequests ?? 0,
1989
2093
  failedRequests: requestMetrics.failedRequests ?? 0,
1990
2094
  requestMetrics,
1991
- uncertainReceipts,
1992
2095
  observations,
1993
2096
  incomplete,
1994
2097
  repositoryIdentity,
@@ -2265,6 +2368,41 @@ function reservationCost(reservation, resource, leases, nowMs) {
2265
2368
  return Number.isFinite(cost) && cost > 0 ? Math.max(0, cost - accounted) : 0;
2266
2369
  }
2267
2370
 
2371
+ function governorAccountingSafe(state, nowMs) {
2372
+ return RATE_RESOURCES.every((resource) =>
2373
+ Number.isSafeInteger(governorChargedCost(state, resource, nowMs)));
2374
+ }
2375
+
2376
+ function governorChargedCost(state, resource, nowMs) {
2377
+ if (!state || !RATE_RESOURCES.includes(resource)) return Number.NaN;
2378
+ let total = state.debt?.[resource]?.unresolvedUnits +
2379
+ state.debt?.[resource]?.quiescentUnits;
2380
+ if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
2381
+ for (const reservation of [state.controlReceipts?.[resource],
2382
+ ...Object.values(state.reservations ?? {})]) {
2383
+ total += reservationCost(reservation, resource, state.leases, nowMs);
2384
+ if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
2385
+ }
2386
+ return total;
2387
+ }
2388
+
2389
+ function governorAvailableForGrant(state, resource, nowMs) {
2390
+ return availableForGrant({ budget: state?.budgets?.[resource], resource, nowMs,
2391
+ chargedCost: governorChargedCost(state, resource, nowMs) });
2392
+ }
2393
+
2394
+ function governorResourceDecision(state, resource, nowMs, cost) {
2395
+ return resourceDecision({ budget: state?.budgets?.[resource], resource, nowMs, cost,
2396
+ chargedCost: governorChargedCost(state, resource, nowMs) });
2397
+ }
2398
+
2399
+ function governorResourceHeld(state, resource, nowMs, pendingBlock = false) {
2400
+ const budget = state?.budgets?.[resource];
2401
+ const decision = governorAvailableForGrant(state, resource, nowMs);
2402
+ return pendingBlock || budget?.blockUntil > nowMs ||
2403
+ decision.mode === "paused" || decision.mode === "open" && decision.spendable <= 0;
2404
+ }
2405
+
2268
2406
  function availableForGrant({
2269
2407
  budget,
2270
2408
  resource = budget?.resource,
@@ -2517,6 +2655,7 @@ function scheduleIntents({
2517
2655
  leases = {},
2518
2656
  budgets = {},
2519
2657
  reservations = [],
2658
+ debtCosts = { core: 0, graphql: 0 },
2520
2659
  lanes = {},
2521
2660
  cursors = {},
2522
2661
  nowMs,
@@ -2551,7 +2690,7 @@ function scheduleIntents({
2551
2690
 
2552
2691
  const chargedTotals = Object.fromEntries(RATE_RESOURCES.map((resource) => [
2553
2692
  resource,
2554
- reservations.reduce(
2693
+ debtCosts[resource] + reservations.reduce(
2555
2694
  (total, reservation) => total + reservationCost(reservation, resource, leases, nowMs),
2556
2695
  0,
2557
2696
  ),
@@ -2694,17 +2833,17 @@ function scheduleIntents({
2694
2833
  }
2695
2834
 
2696
2835
  // ---------- Credential identities and restart-safe coordination ----------
2697
- const IDENTITY_VERSION = 1;
2836
+ const IDENTITY_VERSION = 2;
2698
2837
  const IDENTITY_ATTEMPT_WINDOW_MS = 15 * 60_000;
2699
2838
  const IDENTITY_ATTEMPT_LIMIT = 3;
2700
2839
  const IDENTITY_HOST_ATTEMPT_LIMIT = 12;
2701
2840
  const IDENTITY_MAX_ATTEMPTS = 4096;
2702
2841
  const HTTP_START_GAP_MS = 250;
2703
2842
  const HTTP_MAX_WAITERS = 128;
2704
- // A permit is reclaimable once it cannot still belong to a live request. The
2705
- // owner-death check alone cannot see a process that is stopped, or one whose
2706
- // release lost its lock, so a permit outliving the subprocess timeout it guards
2707
- // is stale by construction rather than by guess.
2843
+ const HTTP_MAX_PERMITS = 3;
2844
+ // Retain the existing bounded permit lease for managed requests. This bounds
2845
+ // unexpired admissions, not surviving OS children after suspension or owner
2846
+ // death; quota receipts separately retain uncertain transport charges.
2708
2847
  const HTTP_PERMIT_MAX_MS = GH_TIMEOUT_MS + 5_000;
2709
2848
  // The longest a primary window can run. An attempt that never obtained reset
2710
2849
  // evidence is still covered once a full window has demonstrably elapsed since it
@@ -2738,11 +2877,15 @@ function retryableCoordination(reason) {
2738
2877
  return RETRYABLE_COORDINATION_REASONS.includes(reason);
2739
2878
  }
2740
2879
 
2741
- // A dead owner releases its permit immediately; a live owner keeps it only for
2742
- // as long as a request could still be running under it. Without the second
2743
- // test a stopped process, or one whose release never won the lock, holds the
2744
- // single machine-wide permit forever and every pane queues behind it.
2880
+ // Managed permit leases retain their existing age/dead-owner reclamation law.
2881
+ // Reclaiming a slot never retires its quota receipt or proves child quiescence.
2745
2882
  function staleHttpPermit(permit, now, kill = process.kill.bind(process)) {
2883
+ // A dead v1 parent does not prove its untracked child ended. Reserve its
2884
+ // one slot until a subsequent verified boot; two managed slots remain usable.
2885
+ if (permit.kind === "legacy") {
2886
+ const bootId = coordinationBootId();
2887
+ return permit.bootId !== null && bootId !== null && permit.bootId !== bootId;
2888
+ }
2746
2889
  if (pidIsDead(permit.pid, kill)) return true;
2747
2890
  return Number.isFinite(permit.startedAt) && now - permit.startedAt > HTTP_PERMIT_MAX_MS;
2748
2891
  }
@@ -2813,8 +2956,8 @@ function emptyIdentityRegistry() {
2813
2956
  return { version: IDENTITY_VERSION, migration: { activated: false, holdUntil: 0 }, identities: {}, attempts: {}, hosts: {} };
2814
2957
  }
2815
2958
 
2816
- function normalizeIdentityRegistry(raw) {
2817
- if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !== IDENTITY_VERSION ||
2959
+ function normalizeIdentityRegistry(raw, version = IDENTITY_VERSION) {
2960
+ if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !== version ||
2818
2961
  !exactKeys(raw.migration, ["activated", "holdUntil"]) || typeof raw.migration.activated !== "boolean" ||
2819
2962
  !Number.isFinite(raw.migration.holdUntil) || raw.migration.holdUntil < 0 ||
2820
2963
  !isRecord(raw.identities) || !isRecord(raw.attempts) || !isRecord(raw.hosts) ||
@@ -2837,13 +2980,20 @@ function normalizeIdentityRegistry(raw) {
2837
2980
  }
2838
2981
  for (const [host, transport] of Object.entries(raw.hosts)) {
2839
2982
  if (!isRecord(transport)) return null;
2840
- const keys = ["lastStartedAt", "cooldownUntil", "permit"];
2983
+ const keys = ["lastStartedAt", "cooldownUntil", version === 1 ? "permit" : "permits"];
2841
2984
  if (Object.hasOwn(transport, "waiters")) keys.push("waiters");
2842
2985
  if (Object.hasOwn(transport, "throttle")) keys.push("throttle");
2843
2986
  if (!normalizeHost(host) || !exactKeys(transport, keys) ||
2844
2987
  !time(transport.lastStartedAt) || !time(transport.cooldownUntil)) return null;
2845
- if (transport.permit !== null && (!exactKeys(transport.permit, ["pid", "nonce", "startedAt"]) ||
2846
- !Number.isSafeInteger(transport.permit.pid) || transport.permit.pid < 1 || !validGovernorId(transport.permit.nonce) || !time(transport.permit.startedAt))) return null;
2988
+ const permits = version === 1 ? (transport.permit === null ? [] : [transport.permit]) : transport.permits;
2989
+ if (!Array.isArray(permits) || permits.length > HTTP_MAX_PERMITS ||
2990
+ new Set(permits.map((permit) => permit?.nonce)).size !== permits.length ||
2991
+ permits.filter((permit) => permit?.kind === "legacy").length > 1 ||
2992
+ permits.some((permit) => !exactKeys(permit, version === 1
2993
+ ? ["pid", "nonce", "startedAt"] : ["pid", "nonce", "startedAt", "kind", "bootId"]) ||
2994
+ !Number.isSafeInteger(permit.pid) || permit.pid < 1 || !validGovernorId(permit.nonce) || !time(permit.startedAt) ||
2995
+ version !== 1 && (!["request", "bootstrap", "legacy"].includes(permit.kind) ||
2996
+ permit.bootId !== null && (typeof permit.bootId !== "string" || permit.bootId.length > 128)))) return null;
2847
2997
  if (!Object.hasOwn(transport, "throttle")) transport.throttle = emptyTransportThrottle();
2848
2998
  if (!isRecord(transport.throttle) || !exactKeys(transport.throttle, ["attempts", "lastAt", "paused"]) ||
2849
2999
  !Number.isSafeInteger(transport.throttle.attempts) || transport.throttle.attempts < 0 ||
@@ -2874,8 +3024,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
2874
3024
  let state;
2875
3025
  if (raw === null) state = emptyIdentityRegistry();
2876
3026
  else {
2877
- try { state = normalizeIdentityRegistry(JSON.parse(raw)); }
2878
- catch { return { ok: false, reason: "corrupt" }; }
3027
+ try {
3028
+ const parsed = JSON.parse(raw);
3029
+ state = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
3030
+ } catch { return { ok: false, reason: "corrupt" }; }
2879
3031
  }
2880
3032
  if (!state) return { ok: false, reason: "corrupt" };
2881
3033
  // Captured before the prunes below so their deletions still count as a
@@ -2883,6 +3035,19 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
2883
3035
  // funnels through here, and each one used to serialize and rename the
2884
3036
  // registry while holding the one machine-wide lock.
2885
3037
  const before = JSON.stringify(state);
3038
+ if (state.version === 1) {
3039
+ // Keep v1 writable until a live owner commits its response and cooldown.
3040
+ // Returning the shared transient reason preserves cached identity/leases.
3041
+ if (Object.values(state.hosts).some((transport) =>
3042
+ transport.permit && !pidIsDead(transport.permit.pid, kill))) return { ok: false, reason: "busy" };
3043
+ for (const transport of Object.values(state.hosts)) {
3044
+ transport.permits = transport.permit
3045
+ ? [{ ...transport.permit, kind: "legacy", bootId: coordinationBootId() }] : [];
3046
+ delete transport.permit;
3047
+ }
3048
+ state.version = IDENTITY_VERSION;
3049
+ }
3050
+
2886
3051
  for (const transport of Object.values(state.hosts)) {
2887
3052
  transport.waiters = transport.waiters.filter((waiter) => waiter.deadline > now && !pidIsDead(waiter.pid, kill));
2888
3053
  }
@@ -2913,9 +3078,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
2913
3078
  // stops the entry from being retained for the life of the install.
2914
3079
  : budget?.resetMs > attempt.startedAt + IDENTITY_UNCERTAIN_MAX_MS;
2915
3080
  if (!attempt.accounted && !resetCovered) return { ok: true, value: false };
2916
- delete ledger.value.reservations[`reservation:${id}`];
2917
- const write = writeGovernorState(scope.path, ledger.value);
2918
- return write.ok ? { ok: true, value: true } : write;
3081
+ // Retiring registry attempt history never retires quota debt. A reset
3082
+ // proves a counter window changed, not that a suspended child cannot
3083
+ // dispatch after it resumes.
3084
+ return { ok: true, value: true };
2919
3085
  });
2920
3086
  if (!retired.ok) return retired;
2921
3087
  if (retired.value) delete state.attempts[id];
@@ -3010,9 +3176,10 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
3010
3176
  return withIdentityRegistry(root, (state) => {
3011
3177
  const migration = inspectLegacyMigration(root, state, now);
3012
3178
  if (!migration.ok) return migration;
3013
- const transport = state.hosts[host] ??= { lastStartedAt: 0, cooldownUntil: 0, permit: null };
3014
- if (transport.permit && staleHttpPermit(transport.permit, now, kill)) transport.permit = null;
3015
- if (transport.permit || transport.waiters?.length) return { ok: false, reason: "identity-busy", retryAt: now + 1000 };
3179
+ const transport = state.hosts[host] ??= emptyIdentityTransport();
3180
+ transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, now, kill));
3181
+ if (transport.throttle.paused) return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
3182
+ if (transport.permits.some((permit) => permit.kind !== "legacy") || transport.waiters.length) return { ok: false, reason: "identity-busy", retryAt: now + 1000 };
3016
3183
  const attempts = Object.values(state.attempts);
3017
3184
  const recent = attempts.filter((attempt) => attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
3018
3185
  const own = recent.filter((attempt) => attempt.credentialKey === credentialKey && attempt.resource === resource && attempt.exceptional);
@@ -3027,7 +3194,7 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
3027
3194
  const nonce = governorId();
3028
3195
  state.attempts[id] = { credentialKey, host, resource, startedAt: now, retryAt: now + attemptRetryDelayMs(own.length),
3029
3196
  ownerPid: process.pid, nonce, status: "started", quotaKey: null, resetMs: null, accounted: false, imported: false, exceptional: true };
3030
- transport.permit = { pid: process.pid, nonce, startedAt: now };
3197
+ transport.permits.push({ pid: process.pid, nonce, startedAt: now, kind: "bootstrap", bootId: null });
3031
3198
  transport.lastStartedAt = now;
3032
3199
  return { ok: true, value: { id, nonce } };
3033
3200
  }, { now, kill });
@@ -3045,32 +3212,31 @@ function importIdentityDebts(root, state, credentialKey, identity, now) {
3045
3212
  return withGovernorLock(scope, () => {
3046
3213
  const loaded = readIdentityQuotaState(state, scope, now);
3047
3214
  if (!loaded.ok) return loaded;
3215
+ for (const id of Object.keys(loaded.value.importMarkers)) {
3216
+ const attempt = state.attempts[id];
3217
+ if (!attempt || attempt.imported && attempt.quotaKey === identity.quotaKey) {
3218
+ delete loaded.value.importMarkers[id];
3219
+ }
3220
+ }
3048
3221
  const transferred = [];
3049
3222
  for (const [id, attempt] of Object.entries(state.attempts)) {
3050
3223
  if (attempt.credentialKey !== credentialKey || attempt.imported) continue;
3051
- const reservationId = `reservation:${id}`;
3052
- loaded.value.reservations[reservationId] ??= {
3224
+ const legacyControl = {
3053
3225
  leaseId: id, intentId: id, costs: { core: 0, graphql: 0, [attempt.resource]: 1 }, actualCosts: null,
3054
3226
  accountedCosts: { core: 0, graphql: 0, [attempt.resource]: attempt.accounted ? 1 : 0 },
3055
3227
  notBefore: attempt.startedAt, status: "started", epochs: { core: null, graphql: null },
3056
3228
  startedAt: attempt.startedAt, completedAt: null, outcome: null,
3057
3229
  };
3230
+ if (!loaded.value.importMarkers[id] &&
3231
+ !transferReceiptToDebt(loaded.value, legacyControl, now, id)) {
3232
+ return { ok: false, reason: "accounting-overflow" };
3233
+ }
3234
+ if (Object.keys(loaded.value.importMarkers).length >= IDENTITY_MAX_ATTEMPTS &&
3235
+ !loaded.value.importMarkers[id]) return { ok: false, reason: "identity-capacity" };
3236
+ loaded.value.importMarkers[id] = true;
3058
3237
  transferred.push(id);
3059
3238
  }
3060
- // importIdentityDebts writes this ledger before the registry transaction
3061
- // that records the mapping commits. A crash in between leaves a charged
3062
- // receipt here whose attempt is gone from the registry, and nothing else
3063
- // ever looks at it again -- it presents only as a permanently smaller
3064
- // budget. Re-bootstrapping the same credential is exactly when it can be
3065
- // recognised, and a full window since it started proves a reset covered it.
3066
- for (const key of Object.keys(loaded.value.reservations)) {
3067
- const attemptId = /^reservation:(.+)$/.exec(key)?.[1];
3068
- if (!attemptId || state.attempts[attemptId]) continue;
3069
- const reservation = loaded.value.reservations[key];
3070
- if (reservation.startedAt + IDENTITY_UNCERTAIN_MAX_MS > now) continue;
3071
- delete loaded.value.reservations[key];
3072
- }
3073
- const written = writeGovernorState(scope.path, loaded.value);
3239
+ const written = writeGovernorQuotaState(scope.path, loaded.value, now);
3074
3240
  if (!written.ok) return written;
3075
3241
  // Marked transferred only once the receipts are durable. withIdentityRegistry
3076
3242
  // persists the registry whatever this returns, so setting these before the
@@ -3090,7 +3256,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
3090
3256
  const attempt = state.attempts[id];
3091
3257
  if (!attempt || attempt.nonce !== nonce || attempt.credentialKey !== credentialKey) return { ok: false, reason: "stale" };
3092
3258
  const transport = state.hosts[attempt.host];
3093
- if (transport.permit?.nonce === nonce) transport.permit = null;
3259
+ transport.permits = transport.permits.filter((permit) => permit.nonce !== nonce);
3094
3260
  attempt.status = response ? "finished" : "uncertain";
3095
3261
  const parsed = response?.body;
3096
3262
  const evidence = response?.rateLimit;
@@ -3128,7 +3294,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
3128
3294
  ledger.value.epochs.core = observation.budget.epoch;
3129
3295
  ledger.value.observers.core = { etag: typeof response.etag === "string" ? response.etag : null, outcome: "healthy", at: now, nextAt: now + BUDGET_PROBE_MS };
3130
3296
  }
3131
- return writeGovernorState(scope.path, ledger.value);
3297
+ return writeGovernorQuotaState(scope.path, ledger.value, now);
3132
3298
  });
3133
3299
  if (!seeded.ok) return seeded;
3134
3300
  }
@@ -3267,35 +3433,53 @@ function startIdentityControl(coordinator, now = Date.now()) {
3267
3433
  const exceptional = !budget || now >= budget.resetMs + BUDGET_RESET_GRACE_MS;
3268
3434
  if (budget && now >= budget.resetMs && !exceptional) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
3269
3435
  if (!exceptional) {
3270
- const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now), 0);
3436
+ const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now),
3437
+ state.debt.core.unresolvedUnits + state.debt.core.quiescentUnits +
3438
+ reservationCost(state.controlReceipts.core, "core", state.leases, now));
3271
3439
  if (budget.remaining - resourceReserve(budget.limit) - reserved < 1) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
3272
3440
  }
3273
3441
  const recent = Object.values(registry.attempts).filter((attempt) => attempt.credentialKey === identity.credentialKey && attempt.resource === "core" && attempt.exceptional && attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
3274
3442
  const retryAt = Math.max(0, ...recent.map((attempt) => attempt.retryAt), recent.length >= IDENTITY_ATTEMPT_LIMIT ? Math.min(...recent.map((attempt) => attempt.startedAt)) + IDENTITY_ATTEMPT_WINDOW_MS : 0);
3275
3443
  if (exceptional && retryAt > now) return { ok: false, reason: "identity-backoff", retryAt };
3276
- if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS || Object.keys(state.reservations).length >= GOVERNOR_MAX_RESERVATIONS) return { ok: false, reason: "identity-capacity" };
3444
+ if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS) return { ok: false, reason: "identity-capacity" };
3445
+ const priorControl = state.controlReceipts.core;
3446
+ if (priorControl) {
3447
+ if (!retireControlReceiptToDebt(state, "core", now)) return { ok: false, reason: "accounting-overflow" };
3448
+ }
3277
3449
  const id = governorId();
3278
3450
  registry.attempts[id] = { credentialKey: identity.credentialKey, host: identity.host, resource: "core", startedAt: now,
3279
3451
  retryAt: exceptional ? now + attemptRetryDelayMs(recent.length) : now, ownerPid: process.pid, nonce: governorId(), status: "started",
3280
3452
  quotaKey: identity.quotaKey, resetMs: budget?.resetMs ?? null, accounted: false, imported: true, exceptional };
3281
- state.reservations[`reservation:${id}`] = { leaseId: id, intentId: id, costs: { core: 1, graphql: 0 }, actualCosts: null,
3453
+ state.controlReceipts.core = { leaseId: id, intentId: id, costs: { core: 1, graphql: 0 }, actualCosts: null,
3282
3454
  accountedCosts: { core: 0, graphql: 0 }, notBefore: now, status: "started", epochs: { core: budget?.epoch ?? null, graphql: null },
3283
- startedAt: now, completedAt: null, outcome: null };
3284
- const written = writeGovernorState(scope.path, state);
3285
- return written.ok ? { ok: true, value: { id, identity } } : written;
3455
+ startedAt: now, completedAt: null, outcome: null,
3456
+ receipt: {
3457
+ scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
3458
+ ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
3459
+ deadline: now + ACQUISITION_STARTED_DEADLINE_MS,
3460
+ allowance: { core: 1, graphql: 0 }, dispatches: [],
3461
+ } };
3462
+ const written = writeGovernorQuotaState(scope.path, state, now);
3463
+ return written.ok ? { ok: true, value: { id, identity,
3464
+ receiptCapability: { reservationId: `control:core`, controlResource: "core",
3465
+ leaseId: id, scopeHash: scope.hash,
3466
+ ownerNonce: state.controlReceipts.core.receipt.ownerNonce,
3467
+ generation: state.controlReceipts.core.receipt.generation,
3468
+ deadline: state.controlReceipts.core.receipt.deadline } } } : written;
3286
3469
  });
3287
3470
  }, { now });
3288
3471
  }
3289
3472
 
3290
- function settleIdentityControl(coordinator, control, stdout, now = Date.now()) {
3473
+ function settleIdentityControl(coordinator, control, stdout, now = Date.now(),
3474
+ { neverStarted = false } = {}) {
3291
3475
  return withIdentityRegistry(coordinator.root, (registry) => {
3292
3476
  const attempt = registry.attempts[control.id];
3293
3477
  if (!attempt) return { ok: false, reason: "stale" };
3294
3478
  const parsed = parseGhApiResponse(stdout ?? "");
3295
3479
  const evidence = pickRateLimit(parsed.headers);
3296
3480
  const proven = [200, 304].includes(parsed.status) && evidence?.resource === "core";
3297
- attempt.status = proven ? "finished" : "uncertain";
3298
- attempt.accounted = proven;
3481
+ attempt.status = proven || neverStarted ? "finished" : "uncertain";
3482
+ attempt.accounted = proven || neverStarted;
3299
3483
  if (evidence?.resource === "core") {
3300
3484
  attempt.resetMs = evidence.resetMs;
3301
3485
  if (evidence.remaining === 0) attempt.retryAt = Math.max(attempt.retryAt, evidence.resetMs + BUDGET_RESET_GRACE_MS);
@@ -3304,64 +3488,93 @@ function settleIdentityControl(coordinator, control, stdout, now = Date.now()) {
3304
3488
  return withGovernorLock(scope, () => {
3305
3489
  const loaded = readIdentityQuotaState(registry, scope, now);
3306
3490
  if (!loaded.ok) return loaded;
3307
- const reservation = loaded.value.reservations[`reservation:${control.id}`];
3491
+ const reservation = loaded.value.controlReceipts.core?.intentId === control.id
3492
+ ? loaded.value.controlReceipts.core : null;
3308
3493
  if (!reservation) return { ok: false, reason: "stale" };
3309
- if (proven) {
3310
- const cost = parsed.status === 304 ? 0 : 1;
3494
+ if (proven || neverStarted) {
3495
+ const cost = neverStarted || parsed.status === 304 ? 0 : 1;
3311
3496
  reservation.status = "completed";
3312
3497
  reservation.completedAt = now;
3313
3498
  reservation.outcome = "measured-success";
3314
3499
  reservation.actualCosts = { core: cost, graphql: 0 };
3315
3500
  reservation.accountedCosts = { core: cost, graphql: 0 };
3501
+ loaded.value.controlReceipts.core = null;
3502
+ } else if (!retireControlReceiptToDebt(loaded.value, "core", now)) {
3503
+ return { ok: false, reason: "accounting-overflow" };
3316
3504
  }
3317
- return writeGovernorState(scope.path, loaded.value);
3505
+ return writeGovernorQuotaState(scope.path, loaded.value, now);
3318
3506
  });
3319
3507
  }, { now });
3320
3508
  }
3321
3509
 
3510
+ const HTTP_PERMIT_FAILURE_HOLDS = Object.freeze({
3511
+ "transport-busy": "coordination",
3512
+ "throttle-paused": "secondary",
3513
+ });
3514
+
3515
+ function httpPermitFailure(reason) {
3516
+ const error = new Error(identityCoordinationMessage(reason));
3517
+ if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, reason)) error.coordinationReason = reason;
3518
+ return error;
3519
+ }
3520
+
3322
3521
  async function acquireIdentityHttpPermit(coordinator, {
3323
3522
  signal,
3324
3523
  now = Date.now,
3325
3524
  wait = (delay) => new Promise((resolve) => setTimeout(resolve, delay)),
3525
+ operationDeadline,
3326
3526
  } = {}) {
3327
3527
  const flushed = await coordinator.flushCompletions?.();
3328
- if (flushed && !flushed.ok) throw new Error(identityCoordinationMessage(flushed.reason));
3528
+ if (flushed && !flushed.ok) throw httpPermitFailure(flushed.reason);
3329
3529
  const identity = coordinator.current();
3330
3530
  if (!identity) throw new Error("Verified identity unavailable");
3331
3531
  const queuedAt = now();
3332
- const deadline = queuedAt + GH_TIMEOUT_MS;
3532
+ const deadline = Number.isFinite(operationDeadline)
3533
+ ? Math.min(operationDeadline, queuedAt + ACQUISITION_STARTED_DEADLINE_MS)
3534
+ : queuedAt + GH_TIMEOUT_MS;
3333
3535
  const nonce = governorId();
3334
3536
  let acquired = false;
3335
3537
  try {
3336
3538
  for (;;) {
3337
3539
  if (signal?.aborted) throw new Error("Request cancelled");
3338
3540
  if (coordinator.current()?.accessKey !== identity.accessKey) throw new Error("Credential changed before request start");
3339
- if (now() >= deadline) throw new Error(identityCoordinationMessage("transport-busy"));
3541
+ if (now() >= deadline) throw httpPermitFailure("transport-busy");
3340
3542
  const result = withIdentityRegistry(coordinator.root, (state) => {
3341
3543
  const migration = inspectLegacyMigration(coordinator.root, state, now());
3342
3544
  if (!migration.ok) return migration;
3343
- if (now() >= deadline) return { ok: false, reason: "transport-busy", retryAt: deadline };
3344
- const transport = state.hosts[identity.host] ??= { lastStartedAt: 0, cooldownUntil: 0, permit: null, waiters: [], throttle: emptyTransportThrottle() };
3545
+ const permitAt = now();
3546
+ if (permitAt >= deadline) return { ok: false, reason: "transport-busy", retryAt: deadline };
3547
+ const transport = state.hosts[identity.host] ??= emptyIdentityTransport();
3345
3548
  // Five consecutive throttles means the ladder has stopped being useful.
3346
3549
  // Waiting out the deadline is what produced the last one, so the pause
3347
3550
  // holds until a person retries or a primary reset opens a recovery.
3348
3551
  if (transport.throttle?.paused) {
3349
3552
  return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
3350
3553
  }
3351
- if (transport.permit && staleHttpPermit(transport.permit, now())) transport.permit = null;
3554
+ transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, permitAt));
3352
3555
  // Admission can happen during the mandatory start gap. Persist call
3353
3556
  // order once so later polling cannot overtake an earlier caller.
3354
- if (!transport.waiters.some((waiter) => waiter.nonce === nonce)) {
3557
+ let waiter = transport.waiters.find((item) => item.nonce === nonce);
3558
+ if (!waiter) {
3355
3559
  if (transport.waiters.length >= HTTP_MAX_WAITERS) return { ok: false, reason: "identity-capacity" };
3356
- transport.waiters.push({ pid: process.pid, nonce, queuedAt, deadline });
3560
+ waiter = { pid: process.pid, nonce, queuedAt: permitAt, deadline: Math.min(deadline, permitAt + GH_TIMEOUT_MS) };
3561
+ transport.waiters.push(waiter);
3562
+ } else if (waiter.deadline - permitAt <= GH_TIMEOUT_MS / 2 && waiter.deadline < deadline) {
3563
+ // Keep the existing registry's short waiter lease and FIFO position.
3564
+ // Renewing this lease cannot extend the admitted operation deadline
3565
+ // above, or the separate timeout of a child that actually starts.
3566
+ waiter.queuedAt = permitAt;
3567
+ waiter.deadline = Math.min(deadline, waiter.queuedAt + GH_TIMEOUT_MS);
3357
3568
  }
3358
3569
  const retryAt = Math.max(transport.cooldownUntil, transport.lastStartedAt + HTTP_START_GAP_MS);
3359
- if (transport.permit || transport.waiters[0].nonce !== nonce || retryAt > now()) {
3360
- return { ok: false, reason: "transport-busy", retryAt: Math.max(retryAt, now() + 50) };
3570
+ if (transport.permits.length >= HTTP_MAX_PERMITS ||
3571
+ transport.permits.some((permit) => permit.kind === "bootstrap") ||
3572
+ transport.waiters[0].nonce !== nonce || retryAt > permitAt) {
3573
+ return { ok: false, reason: "transport-busy", retryAt: Math.max(retryAt, permitAt + 50) };
3361
3574
  }
3362
3575
  transport.waiters.shift();
3363
- transport.permit = { pid: process.pid, nonce, startedAt: now() };
3364
- transport.lastStartedAt = now();
3576
+ transport.permits.push({ pid: process.pid, nonce, startedAt: permitAt, kind: "request", bootId: null });
3577
+ transport.lastStartedAt = permitAt;
3365
3578
  return { ok: true, value: { nonce, host: identity.host } };
3366
3579
  }, { now: now() });
3367
3580
  if (result.ok) { acquired = true; return result.value; }
@@ -3370,7 +3583,7 @@ async function acquireIdentityHttpPermit(coordinator, {
3370
3583
  // retryAt, so give the rest the poll interval the loop already uses.
3371
3584
  const retryAt = result.retryAt ?? now() + 50;
3372
3585
  if ((result.reason !== "transport-busy" && !retryableCoordination(result.reason)) ||
3373
- now() >= deadline || retryAt >= deadline) throw new Error(identityCoordinationMessage(result.reason));
3586
+ now() >= deadline || retryAt >= deadline) throw httpPermitFailure(result.reason);
3374
3587
  await wait(Math.max(1, Math.min(50, retryAt - now())));
3375
3588
  }
3376
3589
  } finally {
@@ -3454,6 +3667,10 @@ function transportCooldownDeadline({ retryAfter, status, secondary, at, attempts
3454
3667
  return status === 429 || secondary === true ? at + throttleLadderMs(attempts) : null;
3455
3668
  }
3456
3669
 
3670
+ function emptyIdentityTransport() {
3671
+ return { lastStartedAt: 0, cooldownUntil: 0, permits: [], waiters: [], throttle: emptyTransportThrottle() };
3672
+ }
3673
+
3457
3674
  function emptyTransportThrottle() {
3458
3675
  return { attempts: 0, lastAt: 0, paused: false };
3459
3676
  }
@@ -3481,10 +3698,21 @@ function clearTransportThrottle(transport) {
3481
3698
  }
3482
3699
  }
3483
3700
 
3701
+ // Completion retries keep their original response time and become no-ops
3702
+ // after a successful durable write. Weak keys do not accumulate tombstones.
3703
+ const identityPermitCompletions = new WeakMap();
3484
3704
  function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date.now()) {
3485
- return withIdentityRegistry(coordinator.root, (state) => {
3705
+ let completion = identityPermitCompletions.get(permit);
3706
+ if (!completion) {
3707
+ completion = { at: now, done: false };
3708
+ identityPermitCompletions.set(permit, completion);
3709
+ }
3710
+ if (completion.done) return { ok: true };
3711
+ now = completion.at;
3712
+ const result = withIdentityRegistry(coordinator.root, (state) => {
3486
3713
  const transport = state.hosts[permit.host];
3487
- if (transport?.permit?.nonce === permit.nonce) transport.permit = null;
3714
+ const held = transport?.permits.find((item) => item.nonce === permit.nonce);
3715
+ if (transport) transport.permits = transport.permits.filter((item) => item.nonce !== permit.nonce);
3488
3716
  if (transport && error) {
3489
3717
  const response = parseGhApiResponse(error.stdout ?? "");
3490
3718
  const verdict = classifyThrottle({
@@ -3503,11 +3731,14 @@ function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date
3503
3731
  at: now,
3504
3732
  });
3505
3733
  }
3506
- } else if (transport && !error) {
3734
+ } else if (transport && !error && held && held.startedAt > transport.throttle.lastAt &&
3735
+ held.startedAt >= transport.cooldownUntil) {
3507
3736
  clearTransportThrottle(transport);
3508
3737
  }
3509
3738
  return { ok: true };
3510
3739
  });
3740
+ if (result.ok) completion.done = true;
3741
+ return result;
3511
3742
  }
3512
3743
 
3513
3744
  // The explicit retry a paused transport waits for. Clears the ladder and the
@@ -3558,11 +3789,11 @@ const GOVERNOR_SCOPE_VERSION = 1;
3558
3789
  // 5: scheduling fairness is persisted, because the starvation it prevents
3559
3790
  // happens *between* planning passes -- each manual refresh is its own pass, so
3560
3791
  // a counter that lives only inside one cannot see a run of them.
3561
- const GOVERNOR_STATE_VERSION = 6;
3792
+ const GOVERNOR_STATE_VERSION = 7;
3562
3793
  // Readable-as-evidence, never written: the shapes a still-running older pane
3563
3794
  // holds. Recognising that such a pane owns a live lease is what the restart
3564
3795
  // boundary depends on.
3565
- const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 5, 4, 3, 2];
3796
+ const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 6, 5, 4, 3, 2];
3566
3797
  // The first version whose file is already in the per-resource observer shape.
3567
3798
  // Anything below it is adapted before it can be read; see readLegacyGovernorState.
3568
3799
  const GOVERNOR_OBSERVER_SPLIT_VERSION = 4;
@@ -3582,6 +3813,8 @@ const MANUAL_GRANT_STREAK_LIMIT = 3;
3582
3813
  const GOVERNOR_MAX_LEASES = 128;
3583
3814
  const GOVERNOR_MAX_INTENTS = 512;
3584
3815
  const GOVERNOR_MAX_RESERVATIONS = 512;
3816
+ const GOVERNOR_MAX_DEBT_GROUPS = 128;
3817
+ const GOVERNOR_MAX_LEDGER_BYTES = 2 * 1024 * 1024;
3585
3818
  const GOVERNOR_LOCK_WAIT_MS = 250;
3586
3819
  // How old an unreadable lock record may be before it counts as abandoned. A
3587
3820
  // creator owns the path from open("wx") until its owner record is written, a
@@ -3697,6 +3930,7 @@ function currentGovernorScope(scope) {
3697
3930
  function emptyGovernorState() {
3698
3931
  return {
3699
3932
  version: GOVERNOR_STATE_VERSION,
3933
+ revision: 0,
3700
3934
  epochs: { core: null, graphql: null },
3701
3935
  budgets: {},
3702
3936
  fairness: { manualStreak: 0 },
@@ -3712,6 +3946,14 @@ function emptyGovernorState() {
3712
3946
  leases: {},
3713
3947
  intents: {},
3714
3948
  reservations: {},
3949
+ controlReceipts: { core: null, graphql: null },
3950
+ debt: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, {
3951
+ unresolvedUnits: 0, unresolvedCount: 0, quiescentUnits: 0, quiescentCount: 0,
3952
+ revision: 0,
3953
+ }])),
3954
+ debtGroups: {},
3955
+ ownerGenerations: {},
3956
+ importMarkers: {},
3715
3957
  manualProbe: null,
3716
3958
  };
3717
3959
  }
@@ -3804,10 +4046,11 @@ function normalizeGovernorIntent(raw, nowMs) {
3804
4046
  return { ...raw, costs };
3805
4047
  }
3806
4048
 
3807
- function normalizeGovernorReservation(raw, nowMs) {
4049
+ function normalizeGovernorReservation(raw, nowMs, { legacy = false } = {}) {
3808
4050
  if (!exactKeys(raw, [
3809
4051
  "leaseId", "intentId", "costs", "actualCosts", "accountedCosts", "notBefore", "status", "epochs",
3810
4052
  "startedAt", "completedAt", "outcome",
4053
+ ...(!legacy ? ["receipt"] : []),
3811
4054
  ])) return null;
3812
4055
  const costs = exactResourceCosts(raw.costs);
3813
4056
  const actualCosts = raw.actualCosts === null ? null : exactResourceCosts(raw.actualCosts);
@@ -3829,13 +4072,46 @@ function normalizeGovernorReservation(raw, nowMs) {
3829
4072
  ) return null;
3830
4073
  if (actualCosts && RATE_RESOURCES.some((resource) => actualCosts[resource] > costs[resource])) return null;
3831
4074
  if (RATE_RESOURCES.some((resource) => accountedCosts[resource] > (actualCosts?.[resource] ?? costs[resource]))) return null;
4075
+ if (!legacy && raw.receipt !== null) {
4076
+ const receipt = raw.receipt;
4077
+ if (!exactKeys(receipt, [
4078
+ "scopeHash", "ownerNonce", "generation", "ownerPid", "ownerBirth", "bootId",
4079
+ "deadline", "allowance", "dispatches",
4080
+ ]) || typeof receipt.scopeHash !== "string" || !/^[0-9a-f]{64}$/.test(receipt.scopeHash) ||
4081
+ !validGovernorId(receipt.ownerNonce) || !validGovernorId(receipt.generation) ||
4082
+ !Number.isSafeInteger(receipt.ownerPid) || receipt.ownerPid < 1 ||
4083
+ receipt.ownerBirth !== null && (typeof receipt.ownerBirth !== "string" || receipt.ownerBirth.length > 128) ||
4084
+ receipt.bootId !== null && (typeof receipt.bootId !== "string" || receipt.bootId.length > 128) ||
4085
+ finiteTimestamp(receipt.deadline, nowMs) === undefined ||
4086
+ !exactKeys(receipt.allowance, RATE_RESOURCES) ||
4087
+ RATE_RESOURCES.some((resource) => receipt.allowance[resource] !== costs[resource]) ||
4088
+ !Array.isArray(receipt.dispatches) || receipt.dispatches.length > 32) return null;
4089
+ const issued = { core: 0, graphql: 0 };
4090
+ for (const [index, dispatch] of receipt.dispatches.entries()) {
4091
+ if (!exactKeys(dispatch, ["sequence", "operation", "costs", "issuedAt", "terminalAt", "childPid", "childBirth", "neverStarted"]) ||
4092
+ dispatch.sequence !== index + 1 || !operationCost(dispatch.operation) ||
4093
+ !exactKeys(dispatch.costs, RATE_RESOURCES) ||
4094
+ RATE_RESOURCES.some((resource) => dispatch.costs[resource] !== operationCost(dispatch.operation)[resource]) ||
4095
+ finiteTimestamp(dispatch.issuedAt, nowMs) === undefined ||
4096
+ finiteTimestamp(dispatch.terminalAt, nowMs, { nullable: true }) === undefined ||
4097
+ dispatch.childPid !== null && (!Number.isSafeInteger(dispatch.childPid) || dispatch.childPid < 1) ||
4098
+ dispatch.childBirth !== null && (typeof dispatch.childBirth !== "string" || dispatch.childBirth.length > 128) ||
4099
+ typeof dispatch.neverStarted !== "boolean" ||
4100
+ dispatch.neverStarted && (dispatch.childPid !== null || dispatch.terminalAt === null)) return null;
4101
+ for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
4102
+ }
4103
+ if (RATE_RESOURCES.some((resource) => !Number.isSafeInteger(issued[resource]) ||
4104
+ issued[resource] > receipt.allowance[resource])) return null;
4105
+ } else if (!legacy && raw.status === "started") {
4106
+ return null;
4107
+ }
3832
4108
  return { ...raw, costs, actualCosts, accountedCosts, epochs };
3833
4109
  }
3834
4110
 
3835
4111
  function normalizeResourceObserver(raw, nowMs) {
3836
4112
  if (!exactKeys(raw, ["etag", "outcome", "at", "nextAt"])) return null;
3837
4113
  if (
3838
- raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0) ||
4114
+ raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0 || raw.etag.length > 512) ||
3839
4115
  !GOVERNOR_PROBE_STATUSES.has(raw.outcome) ||
3840
4116
  finiteTimestamp(raw.at, nowMs) === undefined || raw.at > nowMs ||
3841
4117
  finiteTimestamp(raw.nextAt, nowMs) === undefined
@@ -3843,10 +4119,11 @@ function normalizeResourceObserver(raw, nowMs) {
3843
4119
  return { ...raw };
3844
4120
  }
3845
4121
 
3846
- function normalizeProbeClaim(raw, nowMs) {
4122
+ function normalizeProbeClaim(raw, nowMs, { legacy = false } = {}) {
3847
4123
  if (raw === null) return null;
3848
4124
  if (!exactKeys(raw, [
3849
4125
  "ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds",
4126
+ ...(!legacy ? ["debtSnapshot"] : []),
3850
4127
  ])) {
3851
4128
  return undefined;
3852
4129
  }
@@ -3858,6 +4135,12 @@ function normalizeProbeClaim(raw, nowMs) {
3858
4135
  !Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
3859
4136
  raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
3860
4137
  ) return undefined;
4138
+ const snapshot = raw.debtSnapshot;
4139
+ if (!legacy && (!exactKeys(snapshot, ["units", "count", "barrierAt", "revision"]) ||
4140
+ !Number.isSafeInteger(snapshot.units) || snapshot.units < 0 ||
4141
+ !Number.isSafeInteger(snapshot.count) || snapshot.count < 0 ||
4142
+ finiteTimestamp(snapshot.barrierAt, nowMs) === undefined ||
4143
+ !Number.isSafeInteger(snapshot.revision) || snapshot.revision < 0)) return undefined;
3861
4144
  return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
3862
4145
  }
3863
4146
 
@@ -3878,9 +4161,11 @@ function normalizeManualProbe(raw, nowMs) {
3878
4161
  // never publishes what it reads. Every other caller takes the default, so a
3879
4162
  // version this build does not write is unreadable and fails closed.
3880
4163
  function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVERNOR_STATE_VERSION } = {}) {
4164
+ const legacyShape = acceptVersion < 7;
3881
4165
  if (!exactKeys(raw, [
3882
4166
  "version", "epochs", "budgets", "fairness", "observers", "probeClaims", "leases",
3883
4167
  "intents", "reservations", "manualProbe",
4168
+ ...(!legacyShape ? ["revision", "controlReceipts", "debt", "debtGroups", "ownerGenerations", "importMarkers"] : []),
3884
4169
  ]) || raw.version !== acceptVersion) return null;
3885
4170
  if (!exactKeys(raw.fairness, ["manualStreak"]) ||
3886
4171
  !Number.isSafeInteger(raw.fairness.manualStreak) || raw.fairness.manualStreak < 0) return null;
@@ -3894,6 +4179,75 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3894
4179
  ) return null;
3895
4180
 
3896
4181
  const state = emptyGovernorState();
4182
+ state.version = acceptVersion;
4183
+ if (legacyShape) {
4184
+ delete state.revision;
4185
+ delete state.controlReceipts;
4186
+ delete state.debt;
4187
+ delete state.debtGroups;
4188
+ delete state.ownerGenerations;
4189
+ delete state.importMarkers;
4190
+ } else {
4191
+ if (!Number.isSafeInteger(raw.revision) || raw.revision < 0) return null;
4192
+ state.revision = raw.revision;
4193
+ if (!exactKeys(raw.controlReceipts, RATE_RESOURCES) || !exactKeys(raw.debt, RATE_RESOURCES) ||
4194
+ !isRecord(raw.debtGroups) || Object.keys(raw.debtGroups).length > GOVERNOR_MAX_DEBT_GROUPS ||
4195
+ !isRecord(raw.ownerGenerations) ||
4196
+ Object.keys(raw.ownerGenerations).length > GOVERNOR_MAX_LEASES + GOVERNOR_MAX_RESERVATIONS ||
4197
+ !isRecord(raw.importMarkers) || Object.keys(raw.importMarkers).length > IDENTITY_MAX_ATTEMPTS) return null;
4198
+ for (const [id, marker] of Object.entries(raw.importMarkers)) {
4199
+ if (!validGovernorId(id) || marker !== true) return null;
4200
+ state.importMarkers[id] = true;
4201
+ }
4202
+ for (const [leaseId, generation] of Object.entries(raw.ownerGenerations)) {
4203
+ if (!validGovernorId(leaseId) || !validGovernorId(generation)) return null;
4204
+ state.ownerGenerations[leaseId] = generation;
4205
+ }
4206
+ for (const resource of RATE_RESOURCES) {
4207
+ const debt = raw.debt[resource];
4208
+ if (!exactKeys(debt, ["unresolvedUnits", "unresolvedCount", "quiescentUnits", "quiescentCount", "revision"]) ||
4209
+ Object.values(debt).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
4210
+ state.debt[resource] = { ...debt };
4211
+ // Control receipts are normalized independently from the 512 data slots.
4212
+ if (raw.controlReceipts[resource] !== null) {
4213
+ const control = normalizeGovernorReservation(raw.controlReceipts[resource], nowMs);
4214
+ if (!control) return null;
4215
+ state.controlReceipts[resource] = control;
4216
+ }
4217
+ }
4218
+ for (const [id, group] of Object.entries(raw.debtGroups)) {
4219
+ if (!validGovernorId(id) || !exactKeys(group, [
4220
+ "nonce", "ownerNonce", "ownerPid", "ownerBirth", "bootId", "unknownChild", "sealed",
4221
+ "quiescent", "units", "counts", "at", "barrierAt", "barrierRevisions",
4222
+ ]) || !validGovernorId(group.nonce) ||
4223
+ group.ownerNonce !== null && !validGovernorId(group.ownerNonce) ||
4224
+ group.ownerPid !== null && (!Number.isSafeInteger(group.ownerPid) || group.ownerPid < 1) ||
4225
+ group.ownerBirth !== null && (typeof group.ownerBirth !== "string" || group.ownerBirth.length > 128) ||
4226
+ group.bootId !== null && (typeof group.bootId !== "string" || group.bootId.length > 128) ||
4227
+ typeof group.unknownChild !== "boolean" || typeof group.sealed !== "boolean" ||
4228
+ typeof group.quiescent !== "boolean" || !exactKeys(group.units, RATE_RESOURCES) ||
4229
+ !exactKeys(group.counts, RATE_RESOURCES) ||
4230
+ [...Object.values(group.units), ...Object.values(group.counts)].some((value) =>
4231
+ !Number.isSafeInteger(value) || value < 0) ||
4232
+ finiteTimestamp(group.at, nowMs) === undefined ||
4233
+ finiteTimestamp(group.barrierAt, nowMs, { nullable: true }) === undefined ||
4234
+ !exactKeys(group.barrierRevisions, RATE_RESOURCES) ||
4235
+ Object.values(group.barrierRevisions).some((value) =>
4236
+ !Number.isSafeInteger(value) || value < 0)) return null;
4237
+ state.debtGroups[id] = structuredClone(group);
4238
+ }
4239
+ for (const resource of RATE_RESOURCES) {
4240
+ const totals = { unresolvedUnits: 0, unresolvedCount: 0,
4241
+ quiescentUnits: 0, quiescentCount: 0 };
4242
+ for (const group of Object.values(state.debtGroups)) {
4243
+ const prefix = group.quiescent ? "quiescent" : "unresolved";
4244
+ totals[`${prefix}Units`] += group.units[resource];
4245
+ totals[`${prefix}Count`] += group.counts[resource];
4246
+ }
4247
+ if (Object.entries(totals).some(([field, amount]) =>
4248
+ !Number.isSafeInteger(amount) || amount !== state.debt[resource][field])) return null;
4249
+ }
4250
+ }
3897
4251
  state.epochs = { ...raw.epochs };
3898
4252
  state.fairness = { ...raw.fairness };
3899
4253
  for (const resource of RATE_RESOURCES) {
@@ -3922,7 +4276,7 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3922
4276
  if (Object.keys(state.intents).length > GOVERNOR_MAX_INTENTS) return null;
3923
4277
  for (const [id, rawReservation] of Object.entries(raw.reservations)) {
3924
4278
  if (!id.startsWith("reservation:") || !validGovernorId(id.slice(12))) return null;
3925
- const reservation = normalizeGovernorReservation(rawReservation, nowMs);
4279
+ const reservation = normalizeGovernorReservation(rawReservation, nowMs, { legacy: legacyShape });
3926
4280
  if (!reservation || id !== `reservation:${reservation.intentId}`) return null;
3927
4281
  if (
3928
4282
  !prune || reservation.status === "started" || reservation.status === "completed" ||
@@ -3930,8 +4284,9 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3930
4284
  ) state.reservations[id] = reservation;
3931
4285
  }
3932
4286
  if (Object.keys(state.reservations).length > GOVERNOR_MAX_RESERVATIONS) return null;
4287
+ if (prune && !legacyShape) retireOwnerGenerations(state);
3933
4288
  for (const resource of RATE_RESOURCES) {
3934
- const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs);
4289
+ const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs, { legacy: legacyShape });
3935
4290
  if (claim === undefined) return null;
3936
4291
  state.probeClaims[resource] = claim?.leaseUntil > nowMs ? claim : null;
3937
4292
  }
@@ -3940,6 +4295,17 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3940
4295
  return state;
3941
4296
  }
3942
4297
 
4298
+ function retireOwnerGenerations(state) {
4299
+ const detailedOwners = new Set(Object.values(state.reservations)
4300
+ .filter((reservation) => reservation.receipt !== null)
4301
+ .map((reservation) => reservation.leaseId));
4302
+ for (const leaseId of Object.keys(state.ownerGenerations)) {
4303
+ if (!state.leases[leaseId] && !detailedOwners.has(leaseId)) {
4304
+ delete state.ownerGenerations[leaseId];
4305
+ }
4306
+ }
4307
+ }
4308
+
3943
4309
  // Reads a still-running older pane's file as evidence. A version number alone
3944
4310
  // is not enough to identify a document: versions 2 and 3 share a *shape* that
3945
4311
  // version 4 replaced, so accepting their version against the current key set
@@ -4048,8 +4414,288 @@ function migrateGovernorState(raw, nowMs) {
4048
4414
  : adaptPreSplitGovernorShape(raw);
4049
4415
  if (!shaped) return null;
4050
4416
  if (!isRecord(shaped.fairness)) shaped.fairness = { manualStreak: 0 };
4051
- shaped.version = GOVERNOR_STATE_VERSION;
4052
- return normalizeGovernorState(dropSupersededTabCosts(shaped), nowMs);
4417
+ shaped.version = 6;
4418
+ const legacy = normalizeGovernorState(dropSupersededTabCosts(shaped), nowMs, {
4419
+ prune: false,
4420
+ acceptVersion: 6,
4421
+ });
4422
+ return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
4423
+ }
4424
+
4425
+ function migrateLegacyGovernorReceipts(legacy, nowMs) {
4426
+ const state = { ...legacy, version: GOVERNOR_STATE_VERSION,
4427
+ revision: 0,
4428
+ controlReceipts: { core: null, graphql: null },
4429
+ debt: emptyGovernorState().debt,
4430
+ debtGroups: {},
4431
+ ownerGenerations: {},
4432
+ importMarkers: {},
4433
+ reservations: {},
4434
+ };
4435
+ state.probeClaims = { core: null, graphql: null };
4436
+ for (const resource of RATE_RESOURCES) {
4437
+ if (legacy.probeClaims[resource]) {
4438
+ state.observers[resource] = { ...state.observers[resource], outcome: "idle", nextAt: nowMs };
4439
+ }
4440
+ }
4441
+ const units = { core: 0, graphql: 0 };
4442
+ const counts = { core: 0, graphql: 0 };
4443
+ for (const reservation of Object.values(legacy.reservations)) {
4444
+ for (const resource of RATE_RESOURCES) {
4445
+ const charge = reservationCost(reservation, resource, legacy.leases, nowMs);
4446
+ if (!Number.isSafeInteger(charge) || !Number.isSafeInteger(units[resource] + charge)) return null;
4447
+ units[resource] += charge;
4448
+ if (charge > 0) counts[resource] += 1;
4449
+ }
4450
+ }
4451
+ if (RATE_RESOURCES.some((resource) => units[resource] > 0)) {
4452
+ const nonce = governorId();
4453
+ state.debtGroups[nonce] = {
4454
+ nonce, ownerNonce: null, ownerPid: null, ownerBirth: null, bootId: coordinationBootId(),
4455
+ unknownChild: true, sealed: true, quiescent: false,
4456
+ units, counts, at: nowMs, barrierAt: null,
4457
+ barrierRevisions: { core: 0, graphql: 0 },
4458
+ };
4459
+ for (const resource of RATE_RESOURCES) {
4460
+ state.debt[resource].unresolvedUnits = units[resource];
4461
+ state.debt[resource].unresolvedCount = counts[resource];
4462
+ state.debt[resource].revision = 1;
4463
+ }
4464
+ }
4465
+ return normalizeGovernorState(state, nowMs);
4466
+ }
4467
+
4468
+ const UNKNOWN_DEBT_GROUP_ID = "ffffffff-ffff-4fff-8fff-ffffffffffff";
4469
+
4470
+ let cachedCoordinationBootId;
4471
+ let cachedCoordinationProcessBirth;
4472
+ function coordinationBootId() {
4473
+ if (cachedCoordinationBootId !== undefined) return cachedCoordinationBootId;
4474
+ try {
4475
+ const value = process.platform === "linux"
4476
+ ? readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()
4477
+ : process.platform === "darwin"
4478
+ ? execFileSync("/usr/sbin/sysctl", ["-n", "kern.boottime"], { encoding: "utf8", timeout: 1000 }).trim()
4479
+ : "";
4480
+ cachedCoordinationBootId = value.length > 0 && value.length <= 128 ? value : null;
4481
+ } catch {
4482
+ cachedCoordinationBootId = null;
4483
+ }
4484
+ return cachedCoordinationBootId;
4485
+ }
4486
+
4487
+ function coordinationProcessBirth(pid = process.pid) {
4488
+ if (pid === process.pid && cachedCoordinationProcessBirth !== undefined) {
4489
+ return cachedCoordinationProcessBirth;
4490
+ }
4491
+ let value = null;
4492
+ try {
4493
+ if (process.platform === "linux") {
4494
+ const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
4495
+ const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/);
4496
+ value = fields[19] ?? null; // starttime is field 22, after pid and comm.
4497
+ } else if (process.platform === "darwin") {
4498
+ value = execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], {
4499
+ encoding: "utf8", timeout: 1000,
4500
+ }).trim() || null;
4501
+ }
4502
+ } catch { /* an unavailable birth identity is explicitly unknown */ }
4503
+ if (value && value.length > 128) value = null;
4504
+ if (pid === process.pid) cachedCoordinationProcessBirth = value;
4505
+ return value;
4506
+ }
4507
+
4508
+ function debtGroupTarget(state, receipt, groupId = null) {
4509
+ const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
4510
+ const count = Object.keys(state.debtGroups).length;
4511
+ const owned = groupId ? state.debtGroups[groupId] : null;
4512
+ if (owned && !owned.quiescent && owned.bootId === incomingBootId &&
4513
+ (!receipt.receipt || owned.ownerNonce === null ||
4514
+ owned.ownerNonce === receipt.receipt.ownerNonce)) return groupId;
4515
+ if (groupId && !owned && count < GOVERNOR_MAX_DEBT_GROUPS - 1) return groupId;
4516
+ const shared = Object.entries(state.debtGroups).find(([, group]) =>
4517
+ !group.quiescent && group.ownerNonce === null && group.unknownChild &&
4518
+ group.bootId === incomingBootId);
4519
+ if (shared) return shared[0];
4520
+ if (count >= GOVERNOR_MAX_DEBT_GROUPS) return null;
4521
+ return state.debtGroups[UNKNOWN_DEBT_GROUP_ID] ? governorId() : UNKNOWN_DEBT_GROUP_ID;
4522
+ }
4523
+
4524
+ function transferReceiptToDebt(state, receipt, nowMs, groupId = null) {
4525
+ const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
4526
+ reservationCost(receipt, resource, state.leases, nowMs)]));
4527
+ if (RATE_RESOURCES.every((resource) => units[resource] === 0)) return true;
4528
+ const id = debtGroupTarget(state, receipt, groupId);
4529
+ if (id === null) return false;
4530
+ const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
4531
+ const dedicated = id === groupId;
4532
+ const group = state.debtGroups[id] ?? {
4533
+ nonce: governorId(), ownerNonce: null, ownerPid: null, ownerBirth: null,
4534
+ bootId: incomingBootId,
4535
+ unknownChild: !dedicated || !receipt.receipt, sealed: true, quiescent: false,
4536
+ units: { core: 0, graphql: 0 }, counts: { core: 0, graphql: 0 },
4537
+ at: nowMs, barrierAt: null,
4538
+ barrierRevisions: { core: 0, graphql: 0 },
4539
+ };
4540
+ if (group.quiescent) return false;
4541
+ if (receipt.receipt && dedicated) {
4542
+ if (group.ownerNonce !== null && group.ownerNonce !== receipt.receipt.ownerNonce) return false;
4543
+ group.ownerNonce = receipt.receipt.ownerNonce;
4544
+ group.ownerPid = receipt.receipt.ownerPid;
4545
+ group.ownerBirth = receipt.receipt.ownerBirth;
4546
+ group.bootId = receipt.receipt.bootId;
4547
+ group.unknownChild ||= receipt.receipt.dispatches.some((dispatch) =>
4548
+ dispatch.terminalAt === null);
4549
+ }
4550
+ for (const resource of RATE_RESOURCES) {
4551
+ const amount = units[resource];
4552
+ if (!Number.isSafeInteger(amount) || amount < 0 ||
4553
+ !Number.isSafeInteger(state.debt[resource].unresolvedUnits + amount) ||
4554
+ !Number.isSafeInteger(group.units[resource] + amount) ||
4555
+ !Number.isSafeInteger(state.debt[resource].unresolvedCount + (amount > 0 ? 1 : 0)) ||
4556
+ !Number.isSafeInteger(state.debt[resource].revision + 1) ||
4557
+ !Number.isSafeInteger(group.counts[resource] + (amount > 0 ? 1 : 0))) return false;
4558
+ }
4559
+ for (const resource of RATE_RESOURCES) {
4560
+ const amount = units[resource];
4561
+ state.debt[resource].unresolvedUnits += amount;
4562
+ state.debt[resource].unresolvedCount += amount > 0 ? 1 : 0;
4563
+ state.debt[resource].revision += 1;
4564
+ group.units[resource] += amount;
4565
+ group.counts[resource] += amount > 0 ? 1 : 0;
4566
+ }
4567
+ state.debtGroups[id] = group;
4568
+ return true;
4569
+ }
4570
+
4571
+ function makeDebtGroupQuiescent(state, group, at) {
4572
+ for (const resource of RATE_RESOURCES) {
4573
+ const debt = state.debt[resource];
4574
+ if (debt.unresolvedUnits < group.units[resource] || debt.unresolvedCount < group.counts[resource] ||
4575
+ !Number.isSafeInteger(debt.quiescentUnits + group.units[resource]) ||
4576
+ !Number.isSafeInteger(debt.quiescentCount + group.counts[resource]) ||
4577
+ !Number.isSafeInteger(debt.revision + 1)) return false;
4578
+ }
4579
+ for (const resource of RATE_RESOURCES) {
4580
+ const debt = state.debt[resource];
4581
+ debt.unresolvedUnits -= group.units[resource];
4582
+ debt.unresolvedCount -= group.counts[resource];
4583
+ debt.quiescentUnits += group.units[resource];
4584
+ debt.quiescentCount += group.counts[resource];
4585
+ debt.revision += 1;
4586
+ }
4587
+ group.quiescent = true;
4588
+ group.barrierAt = at;
4589
+ group.barrierRevisions = Object.fromEntries(RATE_RESOURCES.map((resource) => [
4590
+ resource, state.debt[resource].revision,
4591
+ ]));
4592
+ return true;
4593
+ }
4594
+
4595
+ function retireControlReceiptToDebt(state, resource, at) {
4596
+ const control = state.controlReceipts[resource];
4597
+ if (!control) return true;
4598
+ // A control receipt without an issued dispatch cannot have started HTTP.
4599
+ // Removing it also fences a late owner before it can issue a child.
4600
+ if (control.receipt.dispatches.length === 0) {
4601
+ state.controlReceipts[resource] = null;
4602
+ return true;
4603
+ }
4604
+ const generation = control.receipt.generation;
4605
+ if (!transferReceiptToDebt(state, control, at, generation)) return false;
4606
+ const group = state.debtGroups[generation];
4607
+ if (group && !group.unknownChild && group.ownerPid !== null &&
4608
+ group.ownerBirth !== null && group.bootId !== null &&
4609
+ !makeDebtGroupQuiescent(state, group, at)) return false;
4610
+ state.controlReceipts[resource] = null;
4611
+ return true;
4612
+ }
4613
+
4614
+ function reconcileBootDebt(state, at) {
4615
+ const bootId = coordinationBootId();
4616
+ if (bootId === null) return { ok: true, changed: false };
4617
+ let changed = false;
4618
+ for (const group of Object.values(state.debtGroups)) {
4619
+ if (group.quiescent || group.bootId === null || group.bootId === bootId) continue;
4620
+ if (!makeDebtGroupQuiescent(state, group, at)) {
4621
+ return { ok: false, reason: "accounting-overflow" };
4622
+ }
4623
+ changed = true;
4624
+ }
4625
+ return { ok: true, changed };
4626
+ }
4627
+
4628
+ function compactGovernorReceipts(state, nowMs) {
4629
+ const sealed = new Set();
4630
+ const full = Object.keys(state.reservations).length >= GOVERNOR_MAX_RESERVATIONS;
4631
+ let changed = false;
4632
+ const dueGenerations = new Set(Object.values(state.reservations)
4633
+ .filter((reservation) => reservation.receipt && (
4634
+ reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
4635
+ full && reservation.status === "completed"))
4636
+ .map((reservation) => reservation.receipt.generation));
4637
+ // A full debt journal must not prevent a claimed observer from retiring
4638
+ // older quiescent groups. Keep the detailed receipts until space is proven.
4639
+ const candidates = Object.values(state.reservations).filter((reservation) => {
4640
+ const groupId = reservation.receipt?.generation ?? null;
4641
+ return (full && reservation.status === "completed" && !reservation.receipt ||
4642
+ groupId && dueGenerations.has(groupId) &&
4643
+ !(reservation.status === "started" && reservation.receipt.dispatches.length === 0)) &&
4644
+ RATE_RESOURCES.some((resource) =>
4645
+ reservationCost(reservation, resource, state.leases, nowMs) > 0);
4646
+ });
4647
+ if (candidates.length > 0) {
4648
+ const trial = structuredClone(state);
4649
+ for (const reservation of candidates) {
4650
+ const groupId = reservation.receipt?.generation ?? null;
4651
+ if (debtGroupTarget(trial, reservation, groupId) === null) return { ok: true, changed: false };
4652
+ if (!transferReceiptToDebt(trial, reservation, nowMs, groupId)) {
4653
+ return { ok: false, reason: "accounting-overflow" };
4654
+ }
4655
+ }
4656
+ }
4657
+ if (full) {
4658
+ for (const [id, reservation] of Object.entries(state.reservations)) {
4659
+ if (reservation.status !== "completed" || reservation.receipt) continue;
4660
+ if (!transferReceiptToDebt(state, reservation, nowMs)) {
4661
+ return { ok: false, reason: "accounting-overflow" };
4662
+ }
4663
+ delete state.reservations[id];
4664
+ changed = true;
4665
+ }
4666
+ }
4667
+ for (const reservation of Object.values(state.reservations)) {
4668
+ if (reservation.receipt && (
4669
+ reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
4670
+ full && reservation.status === "completed")) {
4671
+ sealed.add(reservation.receipt.generation);
4672
+ }
4673
+ }
4674
+ if (sealed.size === 0) {
4675
+ if (changed) retireOwnerGenerations(state);
4676
+ return { ok: true, changed };
4677
+ }
4678
+ // Seal the complete owner generation. A late completion of one receipt cannot
4679
+ // make another already-authorized child in that generation disappear.
4680
+ for (const generation of sealed) {
4681
+ for (const [id, reservation] of Object.entries(state.reservations)) {
4682
+ if (reservation.receipt?.generation !== generation) continue;
4683
+ if (!(reservation.status === "started" && reservation.receipt.dispatches.length === 0) &&
4684
+ !transferReceiptToDebt(state, reservation, nowMs, generation)) {
4685
+ return { ok: false, reason: "accounting-overflow" };
4686
+ }
4687
+ delete state.reservations[id];
4688
+ state.ownerGenerations[reservation.leaseId] = governorId();
4689
+ }
4690
+ const group = state.debtGroups[generation];
4691
+ if (group && !group.unknownChild && group.ownerPid !== null &&
4692
+ group.ownerBirth !== null && group.bootId !== null &&
4693
+ !makeDebtGroupQuiescent(state, group, nowMs)) {
4694
+ return { ok: false, reason: "accounting-overflow" };
4695
+ }
4696
+ }
4697
+ retireOwnerGenerations(state);
4698
+ return { ok: true, changed: true };
4053
4699
  }
4054
4700
 
4055
4701
  function migrateV1GovernorState(raw, nowMs) {
@@ -4058,7 +4704,7 @@ function migrateV1GovernorState(raw, nowMs) {
4058
4704
  "intents", "reservations", "manualProbe",
4059
4705
  ]) || raw.version !== 1 || !isRecord(raw.budgets) || !isRecord(raw.reservations)) return null;
4060
4706
  const migrated = structuredClone(raw);
4061
- migrated.version = GOVERNOR_STATE_VERSION;
4707
+ migrated.version = 6;
4062
4708
  migrated.observers = { core: { etag: null, outcome: "idle", at: 0, nextAt: 0 } };
4063
4709
  // The v1 core value came from /rate_limit, which is not authoritative. Do not
4064
4710
  // preserve it as bootstrap evidence. GraphQL remains valid.
@@ -4086,7 +4732,8 @@ function migrateV1GovernorState(raw, nowMs) {
4086
4732
  ]));
4087
4733
  migrated.probeClaims = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, null]));
4088
4734
  migrated.fairness = { manualStreak: 0 };
4089
- return normalizeGovernorState(migrated, nowMs);
4735
+ const legacy = normalizeGovernorState(migrated, nowMs, { prune: false, acceptVersion: 6 });
4736
+ return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
4090
4737
  }
4091
4738
 
4092
4739
  function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
@@ -4099,11 +4746,21 @@ function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
4099
4746
  : { ok: false, reason: "unwritable" };
4100
4747
  }
4101
4748
  try {
4749
+ if (Buffer.byteLength(raw) > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "capacity" };
4102
4750
  const parsed = JSON.parse(raw);
4103
4751
  const normalized = normalizeGovernorState(parsed, nowMs);
4104
4752
  if (normalized) return { ok: true, value: normalized };
4105
4753
  const migrated = migrateGovernorState(parsed, nowMs);
4106
4754
  if (!migrated) return { ok: false, reason: "corrupt" };
4755
+ if (persistMigration && parsed.version === 6) {
4756
+ const backupPath = `${path}.pre-v7.backup`;
4757
+ try {
4758
+ writeFileSync(backupPath, raw, { encoding: "utf8", mode: 0o600, flag: "wx" });
4759
+ chmodSync(backupPath, 0o600);
4760
+ } catch (error) {
4761
+ if (error?.code !== "EEXIST") return { ok: false, reason: "unwritable" };
4762
+ }
4763
+ }
4107
4764
  if (!persistMigration) return { ok: true, value: migrated, migrated: true };
4108
4765
  // Read-only callers persist the exact migration while they still hold the
4109
4766
  // scope lock. Mutating callers use this same value for their final write.
@@ -4118,13 +4775,25 @@ function writeGovernorState(path, state) {
4118
4775
  const parent = dirname(path);
4119
4776
  let tempPath = null;
4120
4777
  try {
4778
+ if (state?.version === GOVERNOR_STATE_VERSION) {
4779
+ if (!Number.isSafeInteger(state.revision) || state.revision === Number.MAX_SAFE_INTEGER) {
4780
+ return { ok: false, reason: "accounting-overflow" };
4781
+ }
4782
+ state.revision += 1;
4783
+ }
4784
+ const serialized = serializeGovernorState(state);
4785
+ if (state?.version === GOVERNOR_STATE_VERSION &&
4786
+ Buffer.byteLength(serialized) > GOVERNOR_MAX_LEDGER_BYTES) {
4787
+ return { ok: false, reason: "capacity" };
4788
+ }
4121
4789
  mkdirSync(parent, { recursive: true, mode: 0o700 });
4122
4790
  chmodSync(parent, 0o700);
4123
4791
  tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
4124
- writeFileSync(tempPath, serializeGovernorState(state), { encoding: "utf8", mode: 0o600, flag: "wx" });
4792
+ writeFileSync(tempPath, serialized, { encoding: "utf8", mode: 0o600, flag: "wx" });
4125
4793
  chmodSync(tempPath, 0o600);
4794
+ // The private temporary inode already has mode 0600. Rename is the commit
4795
+ // point: no fallible operation may report failure after state replacement.
4126
4796
  renameSync(tempPath, path);
4127
- chmodSync(path, 0o600);
4128
4797
  return { ok: true };
4129
4798
  } catch {
4130
4799
  if (tempPath !== null) {
@@ -4134,6 +4803,15 @@ function writeGovernorState(path, state) {
4134
4803
  }
4135
4804
  }
4136
4805
 
4806
+ function writeGovernorQuotaState(path, state, nowMs) {
4807
+ const normalized = normalizeGovernorState(state, nowMs, { prune: false });
4808
+ if (!normalized) return { ok: false, reason: "corrupt" };
4809
+ if (!governorAccountingSafe(normalized, nowMs)) {
4810
+ return { ok: false, reason: "accounting-overflow" };
4811
+ }
4812
+ return writeGovernorState(path, normalized);
4813
+ }
4814
+
4137
4815
  function lockOwner(path) {
4138
4816
  try {
4139
4817
  const owner = JSON.parse(readFileSync(path, "utf8"));
@@ -4443,9 +5121,18 @@ function mutateGovernor(scope, nowMs, mutate) {
4443
5121
  const loaded = readGovernorState(scope.path, at, { persistMigration: false });
4444
5122
  if (!loaded.ok) return loaded;
4445
5123
  const state = loaded.value;
5124
+ if (!governorAccountingSafe(state, at)) return { ok: false, reason: "accounting-overflow" };
5125
+ const oldBootReconciled = reconcileBootDebt(state, at);
5126
+ if (!oldBootReconciled.ok) return oldBootReconciled;
5127
+ const compacted = compactGovernorReceipts(state, at);
5128
+ if (!compacted.ok) return compacted;
5129
+ const bootReconciled = reconcileBootDebt(state, at);
5130
+ if (!bootReconciled.ok) return bootReconciled;
4446
5131
  // A mutator can reject without writing. Keep an untouched migration so
4447
5132
  // that even that path upgrades the file once before releasing the lock.
4448
- const migration = loaded.migrated ? structuredClone(state) : null;
5133
+ const migration = loaded.migrated || compacted.changed ||
5134
+ oldBootReconciled.changed || bootReconciled.changed
5135
+ ? structuredClone(state) : null;
4449
5136
  const returnAfterMigration = (result) => {
4450
5137
  if (migration === null) return result;
4451
5138
  const written = writeGovernorState(scope.path, migration);
@@ -4453,6 +5140,7 @@ function mutateGovernor(scope, nowMs, mutate) {
4453
5140
  };
4454
5141
  const value = mutate(state, at);
4455
5142
  if (value?.ok === false && value.write !== true) return returnAfterMigration(value);
5143
+ if (!governorAccountingSafe(state, at)) return returnAfterMigration({ ok: false, reason: "accounting-overflow" });
4456
5144
  const normalized = normalizeGovernorState(state, at);
4457
5145
  if (!normalized) return returnAfterMigration({ ok: false, reason: "corrupt" });
4458
5146
  const written = writeGovernorState(scope.path, normalized);
@@ -4481,7 +5169,7 @@ const GOVERNOR_MAX_ATOMIC_COST = Math.max(...Object.values(OPERATION_COSTS)
4481
5169
  function returnPacingCredit(state, resource, unusedCost, nowMs) {
4482
5170
  const budget = state.budgets[resource];
4483
5171
  if (!budget || !(unusedCost > 0)) return;
4484
- const decision = resourceDecision({ budget, resource, nowMs, cost: 0, chargedCost: 0 });
5172
+ const decision = governorResourceDecision(state, resource, nowMs, 0);
4485
5173
  const callsPerMs = decision?.callsPerMs;
4486
5174
  if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) return;
4487
5175
  const credit = Math.min(unusedCost, GOVERNOR_MAX_ATOMIC_COST) / callsPerMs;
@@ -4544,6 +5232,10 @@ function scheduleGovernorState(state, nowMs) {
4544
5232
  leases: state.leases,
4545
5233
  budgets: state.budgets,
4546
5234
  reservations: Object.entries(state.reservations).map(([id, reservation]) => ({ id, ...reservation })),
5235
+ debtCosts: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
5236
+ state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
5237
+ reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs),
5238
+ ])),
4547
5239
  lanes,
4548
5240
  cursors,
4549
5241
  nowMs,
@@ -4572,6 +5264,7 @@ function scheduleGovernorState(state, nowMs) {
4572
5264
  startedAt: null,
4573
5265
  completedAt: null,
4574
5266
  outcome: null,
5267
+ receipt: null,
4575
5268
  };
4576
5269
  reservationCount += 1;
4577
5270
  delete state.intents[grant.intentId];
@@ -4676,6 +5369,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4676
5369
  dueAtCoreReset,
4677
5370
  );
4678
5371
  if (nextAt > at) return { value: { status: "waiting", nextAt } };
5372
+ if (resource === "graphql" && state.controlReceipts.graphql) {
5373
+ // The old claim's lease has expired. Keep an issued child charged as
5374
+ // debt, and fence the old capability so a new observer can make progress.
5375
+ if (!retireControlReceiptToDebt(state, "graphql", at)) {
5376
+ return { ok: false, reason: "accounting-overflow" };
5377
+ }
5378
+ }
4679
5379
  const nonce = randomUUID();
4680
5380
  const startedReservationIds = Object.entries(state.reservations)
4681
5381
  // An expired owner can no longer settle its request. Keep that uncertain
@@ -4686,6 +5386,36 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4686
5386
  state.leases[reservation.leaseId]?.expiresAt > at &&
4687
5387
  reservationCost(reservation, resource, state.leases, at) > 0)
4688
5388
  .map(([id]) => id);
5389
+ const quiescentGroups = Object.values(state.debtGroups).filter((group) =>
5390
+ group.quiescent && group.barrierAt < at &&
5391
+ (group.units[resource] > 0 || group.counts[resource] > 0));
5392
+ const barrierAt = Math.max(0, ...quiescentGroups.map((group) => group.barrierAt));
5393
+ const debtUnits = quiescentGroups.reduce((sum, group) => sum + group.units[resource], 0);
5394
+ const debtCount = quiescentGroups.reduce((sum, group) => sum + group.counts[resource], 0);
5395
+ if (!Number.isSafeInteger(debtUnits) || !Number.isSafeInteger(debtCount)) {
5396
+ return { ok: false, reason: "accounting-overflow" };
5397
+ }
5398
+ const debtSnapshot = {
5399
+ units: debtUnits,
5400
+ count: debtCount,
5401
+ barrierAt,
5402
+ revision: state.debt[resource].revision,
5403
+ };
5404
+ if (resource === "graphql") {
5405
+ state.controlReceipts.graphql = {
5406
+ leaseId, intentId: nonce, costs: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS },
5407
+ actualCosts: null, accountedCosts: { core: 0, graphql: 0 },
5408
+ notBefore: at, status: "started",
5409
+ epochs: { core: null, graphql: state.epochs.graphql },
5410
+ startedAt: at, completedAt: null, outcome: null,
5411
+ receipt: {
5412
+ scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
5413
+ ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
5414
+ deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
5415
+ allowance: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS }, dispatches: [],
5416
+ },
5417
+ };
5418
+ }
4689
5419
  state.probeClaims[resource] = {
4690
5420
  ownerLeaseId: leaseId,
4691
5421
  nonce,
@@ -4693,6 +5423,7 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4693
5423
  nextAt: at,
4694
5424
  claimAt: at,
4695
5425
  startedReservationIds,
5426
+ debtSnapshot,
4696
5427
  };
4697
5428
  state.observers[resource].outcome = "waiting";
4698
5429
  state.observers[resource].at = at;
@@ -4704,6 +5435,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4704
5435
  leaseUntil: state.probeClaims[resource].leaseUntil,
4705
5436
  startedReservationIds,
4706
5437
  coreEtag: state.observers[resource].etag,
5438
+ ...(resource === "graphql" ? { receiptCapability: {
5439
+ reservationId: "control:graphql", controlResource: "graphql", leaseId,
5440
+ scopeHash: scope.hash,
5441
+ ownerNonce: state.controlReceipts.graphql.receipt.ownerNonce,
5442
+ generation: state.controlReceipts.graphql.receipt.generation,
5443
+ deadline: state.controlReceipts.graphql.receipt.deadline,
5444
+ } } : {}),
4707
5445
  } };
4708
5446
  });
4709
5447
  }
@@ -4785,6 +5523,34 @@ function budgetFromObservation(raw, previous, nowMs, {
4785
5523
  return { status: "accepted", budget: next, epochChanged };
4786
5524
  }
4787
5525
 
5526
+ function retireClaimedQuiescentDebt(state, resource, claim) {
5527
+ const snapshot = claim.debtSnapshot;
5528
+ if (snapshot.units === 0 && snapshot.count === 0) return true;
5529
+ if (claim.claimAt <= snapshot.barrierAt) return true;
5530
+ const eligible = Object.entries(state.debtGroups).filter(([, group]) =>
5531
+ group.quiescent && group.barrierAt < claim.claimAt &&
5532
+ group.barrierRevisions[resource] <= snapshot.revision &&
5533
+ (group.units[resource] > 0 || group.counts[resource] > 0));
5534
+ const units = eligible.reduce((total, [, group]) => total + group.units[resource], 0);
5535
+ const count = eligible.reduce((total, [, group]) => total + group.counts[resource], 0);
5536
+ const debt = state.debt[resource];
5537
+ if (!Number.isSafeInteger(units) || !Number.isSafeInteger(count) ||
5538
+ units !== snapshot.units || count !== snapshot.count ||
5539
+ debt.quiescentUnits < units || debt.quiescentCount < count ||
5540
+ !Number.isSafeInteger(debt.revision + 1)) return false;
5541
+ debt.quiescentUnits -= units;
5542
+ debt.quiescentCount -= count;
5543
+ debt.revision += 1;
5544
+ for (const [id, group] of eligible) {
5545
+ group.units[resource] = 0;
5546
+ group.counts[resource] = 0;
5547
+ if (RATE_RESOURCES.every((name) => group.units[name] === 0 && group.counts[name] === 0)) {
5548
+ delete state.debtGroups[id];
5549
+ }
5550
+ }
5551
+ return true;
5552
+ }
5553
+
4788
5554
  function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
4789
5555
  return mutateGovernor(scope, nowMs, (state, at) => {
4790
5556
  const claim = state.probeClaims[resource];
@@ -4833,6 +5599,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
4833
5599
  }
4834
5600
  }
4835
5601
  if (changedResources.length === 0) return { ok: false, reason: "corrupt" };
5602
+ if (!retireClaimedQuiescentDebt(state, resource, claim)) {
5603
+ return { ok: false, reason: "accounting-overflow" };
5604
+ }
4836
5605
  const completedBeforeClaim = Object.entries(state.reservations)
4837
5606
  .filter(([, reservation]) =>
4838
5607
  reservation.status === "completed" && reservation.completedAt < claim.claimAt);
@@ -4884,6 +5653,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
4884
5653
  state.budgets = nextBudgets;
4885
5654
  state.epochs = nextEpochs;
4886
5655
  state.probeClaims[resource] = null;
5656
+ if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
5657
+ state.controlReceipts.graphql = null;
5658
+ }
4887
5659
  {
4888
5660
  const published = nextBudgets[resource];
4889
5661
  state.observers[resource] = {
@@ -4944,6 +5716,15 @@ function failProbeClaim(scope, leaseId, nonce, nowMs, resource) {
4944
5716
  // also marked core failed would erase authority core had legitimately
4945
5717
  // established, and stall a lane with nothing wrong with it.
4946
5718
  state.probeClaims[resource] = null;
5719
+ if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
5720
+ const control = state.controlReceipts.graphql;
5721
+ if (control.receipt.dispatches.length === 0) state.controlReceipts.graphql = null;
5722
+ else if (control.receipt.dispatches.every((dispatch) => dispatch.terminalAt !== null)) {
5723
+ if (!retireControlReceiptToDebt(state, "graphql", at)) {
5724
+ return { ok: false, reason: "accounting-overflow" };
5725
+ }
5726
+ }
5727
+ }
4947
5728
  state.observers[resource] = {
4948
5729
  ...state.observers[resource],
4949
5730
  outcome: "failed",
@@ -5118,7 +5899,8 @@ function startReservation(scope, reservationId, nowMs) {
5118
5899
  }
5119
5900
  const charged = Object.values(state.reservations).reduce(
5120
5901
  (total, item) => total + reservationCost(item, resource, state.leases, at),
5121
- 0,
5902
+ state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
5903
+ reservationCost(state.controlReceipts[resource], resource, state.leases, at),
5122
5904
  );
5123
5905
  if (budget.remaining - resourceReserve(budget.limit) - charged < 0) {
5124
5906
  reservation.status = "cancelled";
@@ -5127,43 +5909,204 @@ function startReservation(scope, reservationId, nowMs) {
5127
5909
  }
5128
5910
  reservation.status = "started";
5129
5911
  reservation.startedAt = at;
5130
- return { value: { status: "started", reservationId } };
5912
+ const generation = state.ownerGenerations[reservation.leaseId] ?? governorId();
5913
+ state.ownerGenerations[reservation.leaseId] = generation;
5914
+ reservation.receipt = {
5915
+ scopeHash: scope.hash,
5916
+ ownerNonce: generation,
5917
+ generation,
5918
+ ownerPid: process.pid,
5919
+ ownerBirth: coordinationProcessBirth(),
5920
+ bootId: coordinationBootId(),
5921
+ deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
5922
+ allowance: { ...reservation.costs },
5923
+ dispatches: [],
5924
+ };
5925
+ return { value: { status: "started", reservationId,
5926
+ receiptCapability: receiptCapabilityFromReservation(reservationId, reservation) } };
5131
5927
  });
5132
5928
  }
5133
5929
 
5134
- function completeReservation(scope, reservationId, completion, nowMs) {
5930
+ function mutateReceiptDispatch(scope, capability, nowMs, mutate) {
5931
+ if (!capability || capability.scopeHash !== scope?.hash ||
5932
+ !validGovernorId(capability.ownerNonce) || !validGovernorId(capability.generation) ||
5933
+ !validGovernorId(capability.leaseId) ||
5934
+ typeof capability.reservationId !== "string") return { ok: false, reason: "receipt-invalid" };
5135
5935
  return mutateGovernor(scope, nowMs, (state, at) => {
5136
- const reservation = state.reservations[reservationId];
5137
- if (!reservation || reservation.status !== "started" || !GOVERNOR_OUTCOMES.has(completion?.outcome)) {
5138
- return { ok: false, reason: "stale" };
5139
- }
5140
- const suppliedActual = completion.actualCost === undefined
5141
- ? null : exactResourceCosts(completion.actualCost);
5142
- if (completion.actualCost !== undefined && !suppliedActual) return { ok: false, reason: "corrupt" };
5143
- const measured = completion.outcome === "measured-success"
5144
- ? suppliedActual
5145
- : reservation.costs;
5146
- if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
5147
- return { ok: false, reason: "corrupt" };
5148
- }
5149
- reservation.status = "completed";
5150
- reservation.completedAt = at;
5151
- reservation.outcome = completion.outcome;
5152
- reservation.actualCosts = { ...measured };
5153
- reservation.accountedCosts = { core: 0, graphql: 0 };
5154
- // The same rule as the observing settlement path: this one narrows the
5155
- // charge in exactly the same way, so it releases the pacing in exactly the
5156
- // same way. Wiring it into only one of the two settlement paths left every
5157
- // doctor probe and manual operation paying for capacity it did not use.
5158
- if (completion.outcome === "measured-success") {
5159
- for (const resource of RATE_RESOURCES) {
5160
- returnPacingCredit(state, resource, reservation.costs[resource] - measured[resource], at);
5936
+ const controlResource = capability.controlResource;
5937
+ if (controlResource !== undefined && !RATE_RESOURCES.includes(controlResource)) {
5938
+ return { ok: false, reason: "receipt-invalid" };
5939
+ }
5940
+ const reservation = controlResource
5941
+ ? state.controlReceipts[controlResource] : state.reservations[capability.reservationId];
5942
+ const receipt = reservation?.receipt;
5943
+ if (!receipt || receipt.scopeHash !== capability.scopeHash ||
5944
+ receipt.ownerNonce !== capability.ownerNonce || receipt.generation !== capability.generation ||
5945
+ reservation.leaseId !== capability.leaseId || receipt.deadline !== capability.deadline) {
5946
+ return { ok: false, reason: "compacted" };
5947
+ }
5948
+ return mutate(state, reservation, receipt, at);
5949
+ });
5950
+ }
5951
+
5952
+ function issueGovernorDispatch(scope, capability, operation, nowMs) {
5953
+ const costs = operationCost(operation);
5954
+ if (!costs) return { ok: false, reason: "undeclared" };
5955
+ if ([...terminalizationBacklog.keys()].filter((key) =>
5956
+ key.startsWith(`${scope?.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
5957
+ return { ok: false, reason: "completion-capacity" };
5958
+ }
5959
+ return mutateReceiptDispatch(scope, capability, nowMs, (state, reservation, receipt, at) => {
5960
+ if (reservation.status !== "started" || receipt.deadline <= at ||
5961
+ (!capability.controlResource && (
5962
+ state.ownerGenerations[reservation.leaseId] !== receipt.generation ||
5963
+ state.leases[reservation.leaseId]?.expiresAt <= at ||
5964
+ !state.leases[reservation.leaseId]))) return { ok: false, reason: "receipt-expired" };
5965
+ const issued = { core: 0, graphql: 0 };
5966
+ for (const dispatch of receipt.dispatches) {
5967
+ for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
5968
+ }
5969
+ if (RATE_RESOURCES.some((resource) =>
5970
+ !Number.isSafeInteger(issued[resource] + costs[resource]) ||
5971
+ issued[resource] + costs[resource] > receipt.allowance[resource])) {
5972
+ return { ok: false, reason: "receipt-allowance" };
5973
+ }
5974
+ const sequence = receipt.dispatches.length + 1;
5975
+ if (sequence > 32) return { ok: false, reason: "receipt-allowance" };
5976
+ receipt.dispatches.push({ sequence, operation, costs: { ...costs },
5977
+ issuedAt: at, terminalAt: null, childPid: null, childBirth: null, neverStarted: false });
5978
+ return { value: { sequence, deadline: receipt.deadline } };
5979
+ });
5980
+ }
5981
+
5982
+ function markGovernorDispatchChild(scope, capability, sequence, pid, nowMs) {
5983
+ if (!Number.isSafeInteger(pid) || pid < 1) return { ok: false, reason: "child-unknown" };
5984
+ return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
5985
+ (_state, _reservation, receipt) => {
5986
+ const dispatch = receipt.dispatches[sequence - 1];
5987
+ if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
5988
+ dispatch.childPid = pid;
5989
+ dispatch.childBirth = coordinationProcessBirth(pid);
5990
+ return { value: { status: "recorded" } };
5991
+ });
5992
+ }
5993
+
5994
+ function terminalGovernorDispatch(scope, capability, sequence, nowMs, { neverStarted = false } = {}) {
5995
+ return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
5996
+ (_state, _reservation, receipt, at) => {
5997
+ const dispatch = receipt.dispatches[sequence - 1];
5998
+ if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
5999
+ if (neverStarted && dispatch.childPid !== null) return { ok: false, reason: "child-started" };
6000
+ if (dispatch.terminalAt === null) {
6001
+ dispatch.terminalAt = at;
6002
+ dispatch.neverStarted = neverStarted;
5161
6003
  }
5162
- }
5163
- return { value: { status: "completed", actualCosts: reservation.actualCosts } };
6004
+ return { value: { status: "terminal" } };
6005
+ });
6006
+ }
6007
+
6008
+ const activeOwnerDispatches = new Map();
6009
+ function ownerDispatchKey(scope, capability) {
6010
+ return `${scope.hash}:${capability.generation}`;
6011
+ }
6012
+
6013
+ function acknowledgeSealedGeneration(scope, capability, sealedNonce, nowMs) {
6014
+ const key = ownerDispatchKey(scope, capability);
6015
+ if ((activeOwnerDispatches.get(key) ?? 0) !== 0) return { ok: false, reason: "children-active" };
6016
+ return mutateGovernor({ ...scope, identityProvider: null }, nowMs, (state, at) => {
6017
+ const group = state.debtGroups[capability.generation];
6018
+ if (!group) return { value: { status: "absent" } };
6019
+ if (group.nonce !== sealedNonce) return { ok: false, reason: "stale" };
6020
+ if (group.quiescent) return { value: { status: "acknowledged", nonce: group.nonce } };
6021
+ if (!group.ownerNonce || group.ownerNonce !== capability.ownerNonce ||
6022
+ !group.ownerBirth || !group.bootId ||
6023
+ group.ownerPid !== process.pid || group.ownerBirth !== coordinationProcessBirth() ||
6024
+ group.bootId !== coordinationBootId() ||
6025
+ Object.values(state.reservations).some((reservation) =>
6026
+ reservation.receipt?.generation === capability.generation)) {
6027
+ return { ok: false, reason: "quiescence-unverified" };
6028
+ }
6029
+ if (!makeDebtGroupQuiescent(state, group, at)) return { ok: false, reason: "accounting-overflow" };
6030
+ return { value: { status: "acknowledged", nonce: group.nonce } };
5164
6031
  });
5165
6032
  }
5166
6033
 
6034
+ function completeReservation(scope, reservationId, completion, nowMs) {
6035
+ const reservation = inspectGovernor({ ...scope, identityProvider: null }, nowMs);
6036
+ if (!reservation.ok) return reservation;
6037
+ const leaseId = reservation.value.reservations[reservationId]?.leaseId;
6038
+ if (!leaseId) return { ok: false, reason: "stale" };
6039
+ return settleReservationWithBudgetObservations(scope, leaseId, reservationId, {
6040
+ ...completion,
6041
+ ...(completion.actualCost ? { actualCosts: completion.actualCost } : {}),
6042
+ observations: [],
6043
+ }, nowMs);
6044
+ }
6045
+
6046
+ function terminalizeReservation(scope, capability, completion, nowMs) {
6047
+ if (!capability?.reservationId || !capability.leaseId) {
6048
+ return { ok: false, reason: "blocked", status: "blocked" };
6049
+ }
6050
+ const result = settleReservationWithBudgetObservations(
6051
+ { ...scope, identityProvider: null }, capability.leaseId,
6052
+ capability.reservationId, { ...completion, capability }, nowMs,
6053
+ );
6054
+ if (result.ok) return { ok: true, value: { ...result.value, status: "settled" } };
6055
+ const status = ["busy", "unwritable"].includes(result.reason) ? "retryable"
6056
+ : result.reason === "stale" ? "compacted" : "blocked";
6057
+ return { ...result, status };
6058
+ }
6059
+
6060
+ const terminalizationBacklog = new Map();
6061
+ function terminalizationBacklogForResource(backlog, scopeHash, resource) {
6062
+ if (!scopeHash) return 0;
6063
+ let count = 0;
6064
+ for (const item of backlog.values()) {
6065
+ if (item.scope.hash !== scopeHash) continue;
6066
+ if (item.capability?.costs?.[resource] > 0 ||
6067
+ !RATE_RESOURCES.some((name) => item.capability?.costs?.[name] > 0)) count += 1;
6068
+ }
6069
+ return count;
6070
+ }
6071
+ function receiptCapabilityFromReservation(reservationId, reservation) {
6072
+ if (!reservation?.receipt || !reservation?.costs) return null;
6073
+ return {
6074
+ reservationId, leaseId: reservation.leaseId,
6075
+ scopeHash: reservation.receipt.scopeHash,
6076
+ ownerNonce: reservation.receipt.ownerNonce,
6077
+ generation: reservation.receipt.generation,
6078
+ deadline: reservation.receipt.deadline,
6079
+ costs: { ...reservation.costs },
6080
+ };
6081
+ }
6082
+ function queueTerminalization(scope, capability, completion, now = Date.now) {
6083
+ const frozen = { ...scope, identityProvider: null };
6084
+ const key = `${scope.hash}:${capability?.reservationId ?? "invalid"}`;
6085
+ const first = terminalizeReservation(frozen, capability, completion, now());
6086
+ if (first.ok || first.status !== "retryable") return first;
6087
+ if (terminalizationBacklog.has(key)) return first;
6088
+ if ([...terminalizationBacklog.keys()].filter((existing) =>
6089
+ existing.startsWith(`${scope.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
6090
+ return { ok: false, reason: "completion-capacity", status: "blocked" };
6091
+ }
6092
+ const item = { scope: frozen, capability, completion, now, attempt: 0, timer: null };
6093
+ const retry = () => {
6094
+ const result = terminalizeReservation(item.scope, item.capability, item.completion, item.now());
6095
+ if (result.ok || result.status !== "retryable") {
6096
+ terminalizationBacklog.delete(key);
6097
+ return;
6098
+ }
6099
+ const delays = [1000, 2000, 4000, 5000];
6100
+ const delay = delays[Math.min(item.attempt++, delays.length - 1)];
6101
+ item.timer = setTimeout(retry, delay + Math.floor(Math.random() * 100));
6102
+ item.timer.unref?.();
6103
+ };
6104
+ terminalizationBacklog.set(key, item);
6105
+ item.timer = setTimeout(retry, 1000);
6106
+ item.timer.unref?.();
6107
+ return first;
6108
+ }
6109
+
5167
6110
  function settleReservationWithBudgetObservations(
5168
6111
  scope,
5169
6112
  leaseId,
@@ -5173,13 +6116,33 @@ function settleReservationWithBudgetObservations(
5173
6116
  ) {
5174
6117
  return mutateGovernor(scope, nowMs, (state, at) => {
5175
6118
  const reservation = state.reservations[reservationId];
5176
- const lease = state.leases[leaseId];
6119
+ if (reservation?.status === "completed" && reservation.leaseId === leaseId &&
6120
+ (!completion?.capability ||
6121
+ completion.capability.ownerNonce === reservation.receipt?.ownerNonce &&
6122
+ completion.capability.generation === reservation.receipt?.generation)) {
6123
+ return { value: { status: "completed", actualCosts: reservation.actualCosts,
6124
+ accountedCosts: reservation.accountedCosts } };
6125
+ }
5177
6126
  if (
5178
- !lease || lease.expiresAt <= at || !reservation ||
6127
+ !reservation ||
5179
6128
  reservation.leaseId !== leaseId || reservation.status !== "started" ||
5180
6129
  !GOVERNOR_OUTCOMES.has(completion?.outcome)
5181
6130
  ) return { ok: false, reason: "stale" };
5182
- const measured = completion.outcome === "measured-success"
6131
+ if (completion.capability && (completion.capability.scopeHash !== scope.hash ||
6132
+ completion.capability.reservationId !== reservationId ||
6133
+ completion.capability.leaseId !== leaseId ||
6134
+ completion.capability.ownerNonce !== reservation.receipt?.ownerNonce ||
6135
+ completion.capability.generation !== reservation.receipt?.generation ||
6136
+ completion.capability.deadline !== reservation.receipt?.deadline)) {
6137
+ return { ok: false, reason: "receipt-invalid" };
6138
+ }
6139
+ const neverIssued = completion.capability && (
6140
+ completion.neverStarted === true && reservation.receipt.dispatches.length === 0 ||
6141
+ reservation.receipt.dispatches.length > 0 &&
6142
+ reservation.receipt.dispatches.every((dispatch) => dispatch.neverStarted));
6143
+ const outcome = neverIssued ? "measured-success" : completion.outcome;
6144
+ const measured = neverIssued ? { core: 0, graphql: 0 }
6145
+ : outcome === "measured-success"
5183
6146
  ? exactResourceCosts(completion.actualCosts ?? completion.actualCost)
5184
6147
  : reservation.costs;
5185
6148
  if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
@@ -5231,17 +6194,17 @@ function settleReservationWithBudgetObservations(
5231
6194
  }
5232
6195
  reservation.status = "completed";
5233
6196
  reservation.completedAt = at;
5234
- reservation.outcome = completion.outcome;
6197
+ reservation.outcome = outcome;
5235
6198
  reservation.actualCosts = { ...measured };
5236
6199
  reservation.accountedCosts = Object.fromEntries(RATE_RESOURCES.map((resource) => [
5237
6200
  resource,
5238
- completion.outcome === "measured-success"
6201
+ outcome === "measured-success"
5239
6202
  ? Math.min(measured[resource], acceptedCosts[resource])
5240
6203
  : 0,
5241
6204
  ]));
5242
6205
  for (const resource of RATE_RESOURCES) {
5243
6206
  const budget = state.budgets[resource];
5244
- if (!budget || completion.outcome !== "measured-success") continue;
6207
+ if (!budget || outcome !== "measured-success") continue;
5245
6208
  // accountedCosts removes the portion already present in an authoritative
5246
6209
  // counter from the reservation residual. knownLocalUsed contains only
5247
6210
  // that same portion when it arrived after the factor baseline. Their sum
@@ -5255,7 +6218,7 @@ function settleReservationWithBudgetObservations(
5255
6218
  // loss proves nothing, so its worst case stays charged and its pacing stays
5256
6219
  // spent -- refunding there would let a run of timeouts pace as though
5257
6220
  // nothing had been sent.
5258
- if (completion.outcome === "measured-success") {
6221
+ if (outcome === "measured-success") {
5259
6222
  for (const resource of RATE_RESOURCES) {
5260
6223
  returnPacingCredit(state, resource, reservation.costs[resource] - measured[resource], at);
5261
6224
  }
@@ -5341,6 +6304,40 @@ function governorHealth(result, nowMs = Date.now()) {
5341
6304
  };
5342
6305
  }
5343
6306
 
6307
+ // Plain doctor must not call the identity coordinator: even its inspection
6308
+ // path may retire old registry attempts and persist the resulting registry.
6309
+ // A cached identity is useful only when it identifies one quota account for
6310
+ // the requested host. Multiple accounts cannot be resolved without probing.
6311
+ function doctorCachedQuotaScope(host, { root = identityRegistryRoot(), nowMs = Date.now() } = {}) {
6312
+ const registryPath = join(root, "registry.json");
6313
+ let registry;
6314
+ try {
6315
+ const file = lstatSync(registryPath);
6316
+ if (!file.isFile() || file.size > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "unsafe registry" };
6317
+ const parsed = JSON.parse(readFileSync(registryPath, "utf8"));
6318
+ registry = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
6319
+ } catch (error) {
6320
+ return { ok: false, reason: error?.code === "ENOENT" ? "cached identity missing" : "cached registry unreadable" };
6321
+ }
6322
+ if (!registry) return { ok: false, reason: "cached registry corrupt" };
6323
+ const quotas = new Set(Object.values(registry.identities)
6324
+ .filter((identity) => identity.host === host)
6325
+ .map((identity) => identity.quotaKey));
6326
+ if (quotas.size === 0) return { ok: false, reason: "cached identity missing" };
6327
+ if (quotas.size !== 1) return { ok: false, reason: "ambiguous cached identities" };
6328
+ const quotaKey = quotas.values().next().value;
6329
+ const path = join(root, `quota-${quotaKey}.json`);
6330
+ try {
6331
+ if (!lstatSync(path).isFile()) return { ok: false, reason: "unsafe quota path", path };
6332
+ } catch (error) {
6333
+ return { ok: false, reason: error?.code === "ENOENT" ? "cached quota missing" : "cached quota unreadable", path };
6334
+ }
6335
+ const result = readGovernorState(path, nowMs, { persistMigration: false });
6336
+ return result.ok && !result.missing
6337
+ ? { ok: true, result, path, quotaKey }
6338
+ : { ok: false, reason: result.missing ? "cached quota missing" : result.reason, path };
6339
+ }
6340
+
5344
6341
  async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
5345
6342
  let result = run();
5346
6343
  while (!result.ok && result.reason === "busy" && !signal?.aborted && now() < deadline) {
@@ -5470,7 +6467,8 @@ async function refreshResourceBudget(scope, leaseId, signal, resource, {
5470
6467
  let budgets;
5471
6468
  try {
5472
6469
  try { onObserverCall?.(resource); } catch { /* diagnostics cannot block a budget observer */ }
5473
- budgets = await requestIdentityStorage.run(scope, () => readBudgets(signal, scope.host, {
6470
+ budgets = await requestIdentityStorage.run({ ...scope,
6471
+ controlReceiptCapability: claim.value.receiptCapability }, () => readBudgets(signal, scope.host, {
5474
6472
  resources: [resource],
5475
6473
  coreEtag,
5476
6474
  renewClaim: async () => {
@@ -6074,7 +7072,7 @@ const SSH_RECONNECT_STEPS_MS = Object.freeze([1_000, 2_000, 4_000, 8_000, 16_000
6074
7072
  const SSH_STDERR_MAX_BYTES = 4_096;
6075
7073
  const SSH_ALIAS_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$/;
6076
7074
  const COLLECTOR_HOLD_REASONS = new Set([
6077
- "observer", "primary", "secondary", "disconnected", "coordination", "shared-wait", "cache-only",
7075
+ "observer", "primary", "secondary", "disconnected", "app-auth", "coordination", "shared-wait", "cache-only",
6078
7076
  ]);
6079
7077
  const COLLECTOR_ROW_KEYS = Object.freeze({
6080
7078
  actions: ["databaseId", "displayTitle", "workflowName", "number", "headBranch", "status", "conclusion",
@@ -6592,7 +7590,7 @@ function persistGitHubAppIdentity(root, identity, now = Date.now()) {
6592
7590
  const initialized = withGovernorLock(scope, () => {
6593
7591
  const loaded = readIdentityQuotaState(state, scope, now);
6594
7592
  if (!loaded.ok) return loaded;
6595
- return loaded.missing ? writeGovernorState(scope.path, loaded.value) : { ok: true };
7593
+ return loaded.missing ? writeGovernorQuotaState(scope.path, loaded.value, now) : { ok: true };
6596
7594
  });
6597
7595
  if (!initialized.ok) return initialized;
6598
7596
  state.identities[identity.credentialKey] = { host: identity.host, kind: identity.kind,
@@ -8403,7 +9401,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
8403
9401
  if (!["changed", "unchanged"].includes(transition.kind)) {
8404
9402
  const error = new Error("collector received unusable data");
8405
9403
  error.requestMetrics = acquisitionRequestMetrics(result);
8406
- error.uncertainReceipts = result.uncertainReceipts ?? [];
8407
9404
  throw error;
8408
9405
  }
8409
9406
  const rows = transition.kind === "changed" ? transition.data : previous?.rows;
@@ -8435,7 +9432,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
8435
9432
  hold: null,
8436
9433
  capabilities: result.capabilities ?? previous?.capabilities ?? {},
8437
9434
  requestMetrics: acquisitionRequestMetrics(result),
8438
- uncertainReceipts: result.uncertainReceipts ?? [],
8439
9435
  repositoryIdentity: result.repositoryIdentity,
8440
9436
  meta,
8441
9437
  securityNotes: resource === "security"
@@ -8616,6 +9612,7 @@ function createCollectorAcquisitionRuntime({
8616
9612
  },
8617
9613
  });
8618
9614
  const reservationId = admitted.reservationId;
9615
+ const transportScope = { ...context.scope, receiptCapability: admitted.receiptCapability };
8619
9616
  const governor = inspectGovernor(context.scope, now());
8620
9617
  const started = await markStarted({
8621
9618
  reservationId,
@@ -8623,8 +9620,9 @@ function createCollectorAcquisitionRuntime({
8623
9620
  epochs: governor.ok ? governor.value.epochs : {},
8624
9621
  });
8625
9622
  if (!started.ok || started.value?.status !== "started") {
8626
- completeReservation({ ...context.scope, identityProvider: null }, reservationId,
8627
- { outcome: "rejected" }, now());
9623
+ const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
9624
+ { outcome: "rejected", neverStarted: true }, now);
9625
+ if (terminal.status === "blocked") throw new Error("collector receipt settlement blocked");
8628
9626
  throw new Error("collector acquisition start failed");
8629
9627
  }
8630
9628
  const descriptor = tabForKey(resource);
@@ -8636,20 +9634,18 @@ function createCollectorAcquisitionRuntime({
8636
9634
  signal,
8637
9635
  AbortSignal.timeout(ACQUISITION_STARTED_DEADLINE_MS),
8638
9636
  ]);
8639
- let result = await requestIdentityStorage.run(context.scope, () => descriptor.fetch({
9637
+ let result = await requestIdentityStorage.run(transportScope, () => descriptor.fetch({
8640
9638
  signal: boundedSignal,
8641
9639
  entities,
8642
9640
  previousRaw: force ? null : snapshot?.raw ?? null,
8643
9641
  force,
8644
9642
  pages: demand.pages,
8645
- governor: { scope: context.scope, leaseId: context.leaseId },
9643
+ governor: { scope: transportScope, leaseId: context.leaseId },
8646
9644
  ...(securityPolicy ? { sources: securityPolicy.sources } : {}),
8647
9645
  }));
8648
9646
  if (securityPolicy) result = applyGitHubAppSecurityPolicy(result, securityPolicy);
8649
- settleReservationWithBudgetObservations(
8650
- { ...context.scope, identityProvider: null },
8651
- context.leaseId,
8652
- reservationId,
9647
+ const terminal = queueTerminalization(
9648
+ context.scope, admitted.receiptCapability,
8653
9649
  result?.measuredSuccess === false
8654
9650
  ? { outcome: "rejected", observations: result.observations ?? [] }
8655
9651
  : {
@@ -8660,8 +9656,11 @@ function createCollectorAcquisitionRuntime({
8660
9656
  },
8661
9657
  observations: result.observations ?? [],
8662
9658
  },
8663
- now(),
9659
+ now,
8664
9660
  );
9661
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
9662
+ throw new Error(`collector receipt settlement ${terminal.status}`);
9663
+ }
8665
9664
  const completedAt = now();
8666
9665
  const publication = collectorPublicationFromResult(
8667
9666
  resource, result, snapshot, demand, completedAt, claim.state,
@@ -8677,10 +9676,11 @@ function createCollectorAcquisitionRuntime({
8677
9676
  if (reconciliation !== null) publication.nextDueAt = completedAt + reconciliation;
8678
9677
  return publication;
8679
9678
  } catch (error) {
8680
- settleReservationWithBudgetObservations(
8681
- { ...context.scope, identityProvider: null }, context.leaseId, reservationId,
8682
- { outcome: governorOutcomeForError(error), observations: error.budgetObservations ?? [] }, now(),
8683
- );
9679
+ const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
9680
+ { outcome: governorOutcomeForError(error), observations: error.budgetObservations ?? [] }, now);
9681
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
9682
+ error.receiptSettlement = terminal.status;
9683
+ }
8684
9684
  throw error;
8685
9685
  }
8686
9686
  }
@@ -8794,6 +9794,9 @@ function createCollectorAcquisitionRuntime({
8794
9794
  item.nextDueAt = snapshot.nextDueAt;
8795
9795
  if (Number.isSafeInteger(snapshot.generation)) item.lastDeliveredGeneration = snapshot.generation;
8796
9796
  if (Number.isFinite(snapshot.lastSuccessAt)) item.lastDeliveredSuccessAt = snapshot.lastSuccessAt;
9797
+ if (publication && item.requestedAt !== null && snapshot.lastSuccessAt >= item.requestedAt) {
9798
+ item.requestedAt = null;
9799
+ }
8797
9800
  item.onSnapshot(snapshot);
8798
9801
  if (!publication || !Number.isSafeInteger(snapshot.generation)) return true;
8799
9802
  for (const waiter of item.invalidations) {
@@ -8809,11 +9812,17 @@ function createCollectorAcquisitionRuntime({
8809
9812
  }
8810
9813
 
8811
9814
  function startPoll(item, { manual = false, force = false } = {}) {
9815
+ if (closed || item.closed) return null;
9816
+ if (manual && !demandEnabled(item.demand)) item.requestedAt ??= now();
9817
+ if (!item.id) {
9818
+ if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
9819
+ return startInitialize(item);
9820
+ }
8812
9821
  if (item.pollPromise) {
8813
9822
  if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
8814
9823
  return item.pollPromise;
8815
9824
  }
8816
- if (closed || item.closed || !demandEnabled(item.demand)) return null;
9825
+ if (!demandEnabled(item.demand) && item.requestedAt === null) return null;
8817
9826
  item.pollPromise = poll(item, { manual, force }).finally(() => {
8818
9827
  item.pollPromise = null;
8819
9828
  if (item.promotionGeneration !== null) {
@@ -8832,7 +9841,7 @@ function createCollectorAcquisitionRuntime({
8832
9841
  }
8833
9842
 
8834
9843
  function schedule(item, delay, operation = () => startPoll(item)) {
8835
- if (closed || item.closed || !demandEnabled(item.demand)) return;
9844
+ if (closed || item.closed || (!demandEnabled(item.demand) && item.requestedAt === null)) return;
8836
9845
  if (item.timer !== null) clearTimeout_(item.timer);
8837
9846
  item.timer = setTimeout_(() => {
8838
9847
  item.timer = null;
@@ -8898,7 +9907,8 @@ function createCollectorAcquisitionRuntime({
8898
9907
  () => engine.currentClaim(subscriptionId, claim), () => producer({
8899
9908
  target: item.target,
8900
9909
  resource: item.resource,
8901
- demand: { ...ownership.value.demand, background: item.demand.background },
9910
+ demand: { ...ownership.value.demand, background: item.demand.background,
9911
+ active: ownership.value.demand?.active || item.requestedAt !== null },
8902
9912
  snapshot: ownership.value.snapshot,
8903
9913
  claim: { ...claim, state: item },
8904
9914
  provider: item.provider,
@@ -8919,12 +9929,11 @@ function createCollectorAcquisitionRuntime({
8919
9929
  engine.refresh(subscriptionId, { claim, failure: {
8920
9930
  hold: "disconnected",
8921
9931
  requestMetrics: produced.requestMetrics ?? {},
8922
- uncertainReceipts: produced.uncertainReceipts ?? [],
8923
9932
  } });
8924
9933
  engine.unsubscribe(subscriptionId);
8925
9934
  item.id = null;
8926
9935
  targetInitializations.delete(targetInitializationKey(item.target));
8927
- item.onHold?.("disconnected");
9936
+ item.onHold?.("app-auth");
8928
9937
  retryIn = 1;
8929
9938
  return;
8930
9939
  }
@@ -8937,7 +9946,6 @@ function createCollectorAcquisitionRuntime({
8937
9946
  engine.refresh(subscriptionId, { claim, failure: {
8938
9947
  hold: "disconnected",
8939
9948
  requestMetrics: produced.requestMetrics ?? {},
8940
- uncertainReceipts: produced.uncertainReceipts ?? [],
8941
9949
  } });
8942
9950
  item.onHold?.("disconnected");
8943
9951
  retryIn = item.demand.floorMs;
@@ -8960,10 +9968,12 @@ function createCollectorAcquisitionRuntime({
8960
9968
  else if (claim) engine.refresh(subscriptionId, { claim, failure: {
8961
9969
  hold,
8962
9970
  requestMetrics: error?.requestMetrics ?? {},
8963
- uncertainReceipts: error?.uncertainReceipts ?? [],
8964
9971
  } });
8965
9972
  else engine.setHold(subscriptionId, hold, { resource: item.resource, accessKey: item.identity.accessKey });
8966
- item.onHold?.(hold);
9973
+ item.onHold?.(item.provider?.definition?.type === "github-app" &&
9974
+ (error?.apiResponse?.status === 401 || error?.message === "GitHub App token unavailable" ||
9975
+ classify(error) === "auth-problem")
9976
+ ? "app-auth" : hold);
8967
9977
  try { onDiagnostic({ stage: "acquire", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
8968
9978
  retryIn = error?.notStarted && Number.isFinite(error.retryAt)
8969
9979
  ? Math.max(50, error.retryAt - now()) : Math.min(1_000, item.demand.floorMs);
@@ -8979,13 +9989,16 @@ function createCollectorAcquisitionRuntime({
8979
9989
  if (item.id && item.identity?.accessKey) {
8980
9990
  engine.setHold(item.id, hold, { resource: item.resource, accessKey: item.identity.accessKey });
8981
9991
  }
8982
- item.onHold?.(hold);
9992
+ item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
9993
+ error?.message === "GitHub App token unavailable" ? "app-auth" : hold);
8983
9994
  try { onDiagnostic({ stage: "provider-refresh", reason: redact(shortErr(error)) }); } catch { /* isolated */ }
8984
9995
  retryIn = Math.min(1_000, item.demand.floorMs);
8985
9996
  } finally {
8986
9997
  item.polling = false;
8987
9998
  item.pollPhase = null;
8988
- schedule(item, retryIn, item.id ? () => startPoll(item) : () => startInitialize(item));
9999
+ schedule(item, retryIn, item.id
10000
+ ? () => startPoll(item, { manual: item.requestedAt !== null })
10001
+ : () => startInitialize(item));
8989
10002
  }
8990
10003
  }
8991
10004
 
@@ -9018,12 +10031,16 @@ function createCollectorAcquisitionRuntime({
9018
10031
  if (subscription.value.snapshot && !accessReset) {
9019
10032
  deliverCollectorSnapshot(item, subscription.value.snapshot, bindingRevision);
9020
10033
  }
9021
- if (demandEnabled(item.demand)) {
9022
- await startPoll(item, { manual: item.invalidations.size > 0 });
10034
+ if (demandEnabled(item.demand) || item.requestedAt !== null) {
10035
+ const queued = item.queuedRefresh;
10036
+ item.queuedRefresh = null;
10037
+ await startPoll(item, { manual: item.invalidations.size > 0 || item.requestedAt !== null || Boolean(queued),
10038
+ force: queued?.force === true });
9023
10039
  }
9024
10040
  } catch (error) {
9025
10041
  try { onDiagnostic({ stage: "initialize", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
9026
- item.onHold?.("disconnected");
10042
+ item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
10043
+ error?.message === "GitHub App token unavailable" ? "app-auth" : "disconnected");
9027
10044
  schedule(item, 1_000, () => startInitialize(item));
9028
10045
  }
9029
10046
  }
@@ -9040,7 +10057,7 @@ function createCollectorAcquisitionRuntime({
9040
10057
  if (!normalizedDemand) throw new Error("invalid collector demand");
9041
10058
  const item = { target, resource, demand: normalizedDemand, onSnapshot, onHold, id: null, identity: null,
9042
10059
  provider: null, polling: false, pollPhase: null, pollPromise: null, controller: null,
9043
- initializePromise: null, closed: false, timer: null, queuedRefresh: null,
10060
+ initializePromise: null, closed: false, timer: null, queuedRefresh: null, requestedAt: null,
9044
10061
  bindingRevision: 0, invalidations: new Set(), lastDeliveredGeneration: null,
9045
10062
  lastDeliveredSuccessAt: null, promotionGeneration: null, promotionAt: null };
9046
10063
  items.add(item);
@@ -9349,7 +10366,14 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9349
10366
  },
9350
10367
  });
9351
10368
  const write = (message) => {
9352
- if (welcomed && socket?.writable) socket.write(encodeCollectorFrame(message));
10369
+ if (!welcomed || !socket?.writable) return false;
10370
+ socket.write(encodeCollectorFrame(message));
10371
+ return true;
10372
+ };
10373
+ const flushRefresh = (item) => {
10374
+ if (item.pendingRefresh !== null && write({ type: "refresh", id: item.id, force: item.pendingRefresh })) {
10375
+ item.pendingRefresh = null;
10376
+ }
9353
10377
  };
9354
10378
  const resubscribe = (resetGenerations) => {
9355
10379
  for (const item of subscriptions.values()) {
@@ -9359,6 +10383,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9359
10383
  }
9360
10384
  write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
9361
10385
  resource: item.resource, demand: item.demand });
10386
+ flushRefresh(item);
9362
10387
  }
9363
10388
  };
9364
10389
  const connect = () => {
@@ -9449,7 +10474,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9449
10474
  return { ok: false, reason: "invalid" };
9450
10475
  }
9451
10476
  const item = { ...subscription,
9452
- onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1,
10477
+ onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1, pendingRefresh: null,
9453
10478
  sourceAge: createCollectorSourceAgeTracker({ now, monotonicNow, checkpoint: sourceCheckpoint }) };
9454
10479
  subscriptions.set(id, item);
9455
10480
  if (welcomed) write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
@@ -9467,7 +10492,10 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9467
10492
  write({ type: "demand", id, demand: normalized });
9468
10493
  return { ok: true, changed: true };
9469
10494
  },
9470
- refresh(force = false) { write({ type: "refresh", id, force: force === true }); },
10495
+ refresh(force = false) {
10496
+ item.pendingRefresh = force === true || item.pendingRefresh === true;
10497
+ flushRefresh(item);
10498
+ },
9471
10499
  inspect() { write({ type: "inspect", id }); },
9472
10500
  close() { write({ type: "unsubscribe", id }); subscriptions.delete(id); assembler.cancel(id); },
9473
10501
  } };
@@ -9508,6 +10536,7 @@ function createSshCollectorClient({ alias, createTransport = null, random = Math
9508
10536
  }
9509
10537
 
9510
10538
  function collectorDisplayDecision({ connected, hasSnapshot, hold = null }) {
10539
+ if (hold === "app-auth") return { disconnected: true, reason: "app-auth" };
9511
10540
  if (!connected || hold === "disconnected") return { disconnected: true };
9512
10541
  if (hold === "shared-wait") return { mode: "waiting", waitCause: "shared-lane" };
9513
10542
  if (["observer", "primary", "secondary", "coordination"].includes(hold)) {
@@ -9947,8 +10976,10 @@ const ACQUISITION_METRIC_KEYS = [
9947
10976
  const ACQUISITION_HOLD_REASONS = new Set([
9948
10977
  "observer", "primary", "secondary", "disconnected", "coordination",
9949
10978
  ]);
9950
- const ACQUISITION_STORE_VERSION = 1;
10979
+ const ACQUISITION_STORE_VERSION = 2;
9951
10980
  const ACQUISITION_MAX_UNCERTAIN_RECEIPTS = 1024;
10981
+ const ACQUISITION_MAX_QUOTA_PROJECTIONS = 128;
10982
+ const ACQUISITION_QUOTA_PROJECTION_TTL_MS = 60_000;
9952
10983
 
9953
10984
  function normalizeAcquisitionDiagnosticMetadata(raw, path) {
9954
10985
  const normalized = normalizeAcquisitionStore(raw);
@@ -10098,16 +11129,23 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10098
11129
  if (!admitted?.ok || admitted.value.status !== "started") {
10099
11130
  return { index, skipped: { ...skippedDoctorProbe(name, args, admitted), ...(document ? { document } : {}) } };
10100
11131
  }
10101
- return { index, name, args, operation, input, document, reservationId: admitted.value.reservationId };
11132
+ return { index, name, args, operation, input, document,
11133
+ reservationId: admitted.value.reservationId,
11134
+ receiptCapability: admitted.value.receiptCapability };
10102
11135
  });
10103
11136
  results = admittedProbes.map((item) => item.skipped ?? null);
10104
11137
  const runnable = admittedProbes.filter((item) => !item.skipped);
10105
11138
  const settled = await mapAllSettledBounded(runnable, 4, async (item) => {
10106
- const result = await requestIdentityStorage.run(scope, () => probe(item.name, item.args, item.operation, item.input));
11139
+ const result = await requestIdentityStorage.run({ ...scope,
11140
+ receiptCapability: item.receiptCapability }, () => probe(item.name, item.args, item.operation, item.input));
10107
11141
  if (item.document) result.document = item.document;
10108
- completeReservation({ ...scope, identityProvider: null }, item.reservationId, result.failed
11142
+ const terminal = queueTerminalization(scope, item.receiptCapability, result.failed
10109
11143
  ? { outcome: "rejected" }
10110
- : { outcome: "measured-success", actualCost: operationCost(item.operation) }, Date.now());
11144
+ : { outcome: "measured-success", actualCosts: operationCost(item.operation) });
11145
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
11146
+ result.failed = true;
11147
+ result.stderr = `receipt settlement ${terminal.status}`;
11148
+ }
10111
11149
  return result;
10112
11150
  });
10113
11151
  settled.forEach((outcome, index) => {
@@ -10128,9 +11166,32 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10128
11166
  governorResult = inspectGovernor(cleanup, Date.now());
10129
11167
  }
10130
11168
  }
11169
+ const cachedQuota = !probeEndpoints ? doctorCachedQuotaScope(effectiveHost) : null;
10131
11170
  const governor = probeEndpoints
10132
11171
  ? governorHealth(governorResult, Date.now())
10133
- : { status: "not inspected (--probe not requested)", leases: 0, resources: {} };
11172
+ : cachedQuota?.ok ? governorHealth(cachedQuota.result, Date.now())
11173
+ : { status: `unavailable (${cachedQuota?.reason ?? "cached scope unavailable"})`, leases: 0, resources: {} };
11174
+ const diagnosticScope = probeEndpoints
11175
+ ? scopeResult?.ok ? scopeResult.value : null
11176
+ : cachedQuota?.ok ? { path: cachedQuota.path } : null;
11177
+ const recoveryJournal = diagnosticScope ? readRecoveryTransitions(diagnosticScope) : null;
11178
+ const recoveryDetails = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
11179
+ diagnosticScope ? recoveryLedgerSummary(diagnosticScope, resource) : null]));
11180
+ const currentRecovery = RATE_RESOURCES.map((resource) => recoveryJournal?.current[resource])
11181
+ .filter(Boolean).sort((left, right) => right.at - left.at)[0] ?? null;
11182
+ const lastFailedRecovery = [...(recoveryJournal?.entries ?? [])].reverse()
11183
+ .find((entry) => entry.code !== "clear") ?? null;
11184
+ const retryDates = RATE_RESOURCES.flatMap((resource) => {
11185
+ const active = recoveryJournal?.current[resource];
11186
+ const observer = (probeEndpoints ? governorResult : cachedQuota?.result)?.value?.observers[resource];
11187
+ return [active?.retryAt, observer?.outcome === "failed" ? observer.nextAt : null];
11188
+ }).filter((value) => Number.isFinite(value) && value > Date.now());
11189
+ const nextRecoveryAt = retryDates.length > 0 ? Math.min(...retryDates) : null;
11190
+ let recoveryReason = "unavailable";
11191
+ if (currentRecovery) recoveryReason = `${currentRecovery.resource}: ${currentRecovery.code}`;
11192
+ else if (recoveryJournal?.status === "corrupt") recoveryReason = "diagnostic journal corrupt";
11193
+ else if (recoveryJournal?.status === "unavailable") recoveryReason = "diagnostic journal unavailable";
11194
+ else if (["healthy", "missing"].includes(recoveryJournal?.status)) recoveryReason = "none recorded";
10134
11195
  const diagnosticSource = runtime.connect === "local"
10135
11196
  ? "local collector"
10136
11197
  : runtime.connect?.startsWith("ssh:") ? "SSH collector" : "standalone";
@@ -10175,6 +11236,10 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10175
11236
  ),
10176
11237
  "",
10177
11238
  ...section("API governor"),
11239
+ field("cached scope", probeEndpoints ? "verified by probe" : cachedQuota?.ok ? "read-only cached identity" : `unavailable (${cachedQuota?.reason ?? "unknown"})`),
11240
+ ...(cachedQuota?.path && !cachedQuota.ok
11241
+ ? [field("state file", `${join(dirname(cachedQuota.path), "quota-[redacted].json")} (quota fingerprint withheld)`)]
11242
+ : []),
10178
11243
  field("status", governor.status),
10179
11244
  field("live leases", governor.leases),
10180
11245
  ...RATE_RESOURCES.map((resource) => {
@@ -10186,6 +11251,21 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10186
11251
  : "unavailable",
10187
11252
  );
10188
11253
  }),
11254
+ field("recovery reason", recoveryReason),
11255
+ field("last failed transition", lastFailedRecovery?.code ?? "unavailable"),
11256
+ ...RATE_RESOURCES.flatMap((resource) => [
11257
+ field(`${resource} receipt units`, recoveryDetails[resource]?.receiptUnits ?? "unavailable"),
11258
+ field(`${resource} receipt count`, recoveryDetails[resource]?.receiptCount ?? "unavailable"),
11259
+ field(`${resource} debt units`, recoveryDetails[resource]?.debtUnits ?? "unavailable"),
11260
+ field(`${resource} debt count`, recoveryDetails[resource]?.debtCount ?? "unavailable"),
11261
+ field(`${resource} oldest unresolved`, recoveryDetails[resource]?.oldestUnresolvedAt == null
11262
+ ? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].oldestUnresolvedAt))),
11263
+ field(`${resource} observer age`, recoveryDetails[resource]?.observerAt == null
11264
+ ? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].observerAt))),
11265
+ ]),
11266
+ field("next retry", nextRecoveryAt == null ? "unavailable" : new Date(nextRecoveryAt).toISOString()),
11267
+ field("terminalization backlog", currentRecovery?.terminalizationBacklog == null
11268
+ ? "unavailable" : `${currentRecovery.terminalizationBacklog} (last recorded)`),
10189
11269
  "",
10190
11270
  ...section("Acquisition metrics"),
10191
11271
  field("source", acquisitionDiagnostic?.source ?? "standalone"),
@@ -10202,7 +11282,8 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10202
11282
  field("failed requests", acquisitionDiagnostic?.metrics.failedRequests ?? 0),
10203
11283
  field("REST actual 200/304", `${acquisitionDiagnostic?.metrics.rest200 ?? 0}/${acquisitionDiagnostic?.metrics.rest304 ?? 0}`),
10204
11284
  field("proven actual units", `${acquisitionDiagnostic?.metrics.coreUnits ?? 0} core + ${acquisitionDiagnostic?.metrics.graphqlUnits ?? 0} GraphQL`),
10205
- field("conservative outstanding", `${acquisitionDiagnostic?.metrics.uncertainCoreUnits ?? 0} core + ${acquisitionDiagnostic?.metrics.uncertainGraphqlUnits ?? 0} GraphQL`),
11285
+ field("conservative outstanding", acquisitionOutstandingText(acquisitionDiagnostic)),
11286
+ field("legacy unverified", `${acquisitionDiagnostic?.legacyUnverified?.coreUnits ?? 0} core + ${acquisitionDiagnostic?.legacyUnverified?.graphqlUnits ?? 0} GraphQL (diagnostic only)`),
10206
11287
  field("observer calls", acquisitionDiagnostic?.metrics.observerCalls ?? 0),
10207
11288
  field("cache/followers", `${acquisitionDiagnostic?.metrics.cacheHits ?? 0}/${acquisitionDiagnostic?.metrics.joinedFollowers ?? 0}`),
10208
11289
  field("queue wait (measured)", `${acquisitionDiagnostic?.metrics.queueWaitMs ?? 0}ms`),
@@ -10321,7 +11402,7 @@ function parseRepoTarget(value) {
10321
11402
  // in-flight guard absorbs every other one and the effective rate is whatever
10322
11403
  // `gh` can sustain -- the requested interval silently stops being real. Clamping
10323
11404
  // with a stated minimum is honest where silently accepting it would not be.
10324
- const BACKGROUND_MODES = ["all", "off"];
11405
+ const BACKGROUND_MODES = ["all", "off", "on-demand"];
10325
11406
  const MIN_REFRESH_SECONDS = 2;
10326
11407
  const MAX_REFRESH_SECONDS = 3600;
10327
11408
 
@@ -10551,7 +11632,11 @@ Options:
10551
11632
  and running Actions are checked every ${POLL_ACTIVE_CI_MS / 1000}s.
10552
11633
  Safe shared grants may run later.
10553
11634
  --tab <name> Tab to start on: ${TAB_KEYS.join(", ")}.
10554
- --background <mode> ${BACKGROUND_MODES.join(" or ")} (default all). \`off\` never requests data
11635
+ --background <mode> ${BACKGROUND_MODES.join(" or ")} (default on-demand).
11636
+ on-demand watches active Actions; other tabs refresh
11637
+ when opened or with r, then keep cached results.
11638
+ all polls every tab; off polls the selected tab.
11639
+ \`off\` never requests data
10555
11640
  for a tab you are not looking at; its count stays at
10556
11641
  the last known value and visibly ages.
10557
11642
  --verbose Write one line per gh invocation to stderr. stderr
@@ -10579,9 +11664,11 @@ Run it from inside a locally cloned GitHub repository; the repo is inferred
10579
11664
  from the git remote, the same way \`gh\` does it. Requires the \`gh\` CLI
10580
11665
  (2.20 or newer), authenticated via \`gh auth login\`.
10581
11666
 
10582
- On a healthy single pane, the active tab is considered every ${REFRESH_MS / 1000}s while its
11667
+ By default, active Actions is considered every ${REFRESH_MS / 1000}s while its
10583
11668
  content is changing, then every ${POLL_QUIET_MS.actions / 1000}s once two checks in a row come back
10584
- unchanged; each background tab is considered about every ${Math.max(REFRESH_MS * BACKGROUND_EVERY, POLL_BACKGROUND_MS.actions) / 1000}s. Quota-consuming
11669
+ unchanged. Secondary tabs check on opening or r and retain cached results.
11670
+ With --background all, inactive tabs are considered about every
11671
+ ${Math.max(REFRESH_MS * BACKGROUND_EVERY, POLL_BACKGROUND_MS.actions) / 1000}s (Security is slower). Quota-consuming
10585
11672
  calls still need a shared, resource-specific grant that preserves a hard reserve.
10586
11673
  Scheduled checks and manual refresh do not bypass that safety check: neither
10587
11674
  refresh key can spend past the reserve.
@@ -12326,6 +13413,8 @@ const ACQUISITION_MAX_ENTITY_BYTES = 1024 * 1024;
12326
13413
  const ACQUISITION_MAX_ENTITIES = 512;
12327
13414
  const ACQUISITION_MAX_LIVE_TARGETS = 32;
12328
13415
  const ACQUISITION_MAX_SUBSCRIPTIONS = 128;
13416
+ const ACQUISITION_READ_MAX_ATTEMPTS = 24;
13417
+ const ACQUISITION_READ_MAX_MS = 1_500;
12329
13418
  let acquisitionTempSequence = 0;
12330
13419
 
12331
13420
  function acquisitionStorePath(options = {}) {
@@ -12337,7 +13426,8 @@ function emptyAcquisitionStore() {
12337
13426
  version: ACQUISITION_STORE_VERSION,
12338
13427
  producerEpoch: governorId(),
12339
13428
  metrics: emptyAcquisitionMetrics(),
12340
- uncertainReceipts: {},
13429
+ quotaProjections: {},
13430
+ legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
12341
13431
  subscriptions: {},
12342
13432
  queries: {},
12343
13433
  aliases: {},
@@ -12365,32 +13455,6 @@ function acquisitionMetricState(state) {
12365
13455
  return state.metrics;
12366
13456
  }
12367
13457
 
12368
- function removeAcquisitionReceipt(state, id) {
12369
- const receipt = state.uncertainReceipts?.[id];
12370
- if (!receipt) return false;
12371
- const metrics = acquisitionMetricState(state);
12372
- const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
12373
- metrics[key] = Math.max(0, metrics[key] - receipt.units);
12374
- delete state.uncertainReceipts[id];
12375
- for (const record of Object.values(state.queries ?? {})) {
12376
- if (record.claim?.receiptIds?.includes(id)) {
12377
- record.claim.receiptIds = record.claim.receiptIds.filter((receiptId) => receiptId !== id);
12378
- }
12379
- }
12380
- return true;
12381
- }
12382
-
12383
- function addAcquisitionReceipt(state, receipt) {
12384
- state.uncertainReceipts ??= {};
12385
- if (state.uncertainReceipts[receipt.id]) return true;
12386
- if (Object.keys(state.uncertainReceipts).length >= ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return false;
12387
- state.uncertainReceipts[receipt.id] = receipt;
12388
- const metrics = acquisitionMetricState(state);
12389
- const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
12390
- metrics[key] += receipt.units;
12391
- return true;
12392
- }
12393
-
12394
13458
  function normalizeAcquisitionReceipt(raw, expectedId = raw?.id) {
12395
13459
  if (!isRecord(raw) || raw.id !== expectedId || typeof raw.id !== "string" || raw.id.length > 240 ||
12396
13460
  typeof raw.reservationId !== "string" || raw.reservationId.length < 1 ||
@@ -12415,63 +13479,6 @@ function addAcquisitionRequestMetrics(state, delta) {
12415
13479
  return addAcquisitionMetrics(state, observed);
12416
13480
  }
12417
13481
 
12418
- function releaseAcquisitionUncertainty(state, record) {
12419
- if (!record?.claim?.started) return;
12420
- const receiptIds = record.claim.receiptIds ?? [];
12421
- if (Object.hasOwn(record.claim, "receiptIds")) {
12422
- for (const id of receiptIds) removeAcquisitionReceipt(state, id);
12423
- return;
12424
- }
12425
- const costs = tabRequestCost(record.query.resource);
12426
- const metrics = acquisitionMetricState(state);
12427
- metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - costs.core);
12428
- metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - costs.graphql);
12429
- }
12430
-
12431
- function releaseKnownAcquisitionUncertainty(state, record, requestMetrics) {
12432
- if (!record?.claim?.started) return;
12433
- const receiptIds = record.claim.receiptIds ?? [];
12434
- if (Object.hasOwn(record.claim, "receiptIds")) {
12435
- for (const resource of RATE_RESOURCES) {
12436
- const metricKey = resource === "core" ? "coreUnits" : "graphqlUnits";
12437
- const uncertainKey = resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
12438
- const resourceReceipts = receiptIds
12439
- .map((id) => state.uncertainReceipts?.[id])
12440
- .filter((receipt) => receipt?.resource === resource);
12441
- if (Object.hasOwn(requestMetrics, uncertainKey)) {
12442
- let remaining = requestMetrics[uncertainKey];
12443
- for (const receipt of resourceReceipts) {
12444
- const retained = Math.min(receipt.units, remaining);
12445
- const released = receipt.units - retained;
12446
- remaining -= retained;
12447
- if (retained === 0) removeAcquisitionReceipt(state, receipt.id);
12448
- else if (released > 0) {
12449
- receipt.units = retained;
12450
- acquisitionMetricState(state)[uncertainKey] -= released;
12451
- }
12452
- }
12453
- continue;
12454
- }
12455
- if (Object.hasOwn(requestMetrics, metricKey)) {
12456
- for (const receipt of resourceReceipts) removeAcquisitionReceipt(state, receipt.id);
12457
- }
12458
- }
12459
- return;
12460
- }
12461
- const declared = tabRequestCost(record.query.resource);
12462
- const metrics = acquisitionMetricState(state);
12463
- if (Object.hasOwn(requestMetrics, "uncertainCoreUnits")) {
12464
- metrics.uncertainCoreUnits = Math.min(declared.core, requestMetrics.uncertainCoreUnits);
12465
- } else if (Object.hasOwn(requestMetrics, "coreUnits")) {
12466
- metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - declared.core);
12467
- }
12468
- if (Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
12469
- metrics.uncertainGraphqlUnits = Math.min(declared.graphql, requestMetrics.uncertainGraphqlUnits);
12470
- } else if (Object.hasOwn(requestMetrics, "graphqlUnits")) {
12471
- metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - declared.graphql);
12472
- }
12473
- }
12474
-
12475
13482
  function settleAcquisitionWaiters(state, queryKey, generation, at) {
12476
13483
  let queueWaitMs = 0;
12477
13484
  for (const subscription of Object.values(state.subscriptions)) {
@@ -12490,8 +13497,6 @@ function settleInvalidAcquisitionPublication(state, record, claim, requestMetric
12490
13497
  ...normalized,
12491
13498
  failedRequests: Math.max(1, normalized.failedRequests ?? 0),
12492
13499
  });
12493
- releaseKnownAcquisitionUncertainty(state, record, normalized);
12494
- for (const receipt of receipts ?? []) addAcquisitionReceipt(state, receipt);
12495
13500
  settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, at);
12496
13501
  record.claim = null;
12497
13502
  record.hold = { reason: "primary", at };
@@ -12528,9 +13533,18 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
12528
13533
  subscriptionsByQuery.set(subscription.queryKey, grouped);
12529
13534
  }
12530
13535
  const activeQueryKeys = new Set(subscriptions.map((subscription) => subscription.queryKey));
13536
+ const outstandingQuota = Object.entries(state?.quotaProjections ?? {}).map(([scopeHash, projection]) => {
13537
+ const current = projection.status === "current" && projection.at <= nowMs &&
13538
+ nowMs - projection.at <= ACQUISITION_QUOTA_PROJECTION_TTL_MS;
13539
+ return { scopeHash, revision: projection.revision,
13540
+ status: current ? "current" : "unavailable", units: current ? projection.units : null,
13541
+ at: projection.at, accessKeys: projection.accessKeys };
13542
+ });
12531
13543
  const queries = Object.values(state?.queries ?? {}).map((record) => {
12532
13544
  const snapshot = record.snapshot;
12533
13545
  const consumers = subscriptionsByQuery.get(record.query.queryKey) ?? [];
13546
+ const projection = outstandingQuota.find((item) =>
13547
+ item.accessKeys.includes(record.query.accessKey) && item.status === "current");
12534
13548
  return {
12535
13549
  resource: record.query.resource,
12536
13550
  lastSuccessAt: snapshot?.lastSuccessAt ?? null,
@@ -12543,8 +13557,9 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
12543
13557
  graphql: state?.metrics?.graphqlUnits ?? 0,
12544
13558
  },
12545
13559
  uncertainCost: {
12546
- core: state?.metrics?.uncertainCoreUnits ?? 0,
12547
- graphql: state?.metrics?.uncertainGraphqlUnits ?? 0,
13560
+ status: projection ? "current" : "unavailable",
13561
+ core: projection?.units.core ?? null,
13562
+ graphql: projection?.units.graphql ?? null,
12548
13563
  },
12549
13564
  coalescedConsumers: consumers.length,
12550
13565
  };
@@ -12556,12 +13571,31 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
12556
13571
  activeQueries: activeQueryKeys.size,
12557
13572
  activeSubscribers: subscriptions.length,
12558
13573
  metrics: normalizeAcquisitionMetrics(state?.metrics) ?? emptyAcquisitionMetrics(),
13574
+ outstandingQuota: outstandingQuota.map(({ accessKeys: _accessKeys, ...projection }) => projection),
13575
+ legacyUnverified: state?.legacyUnverified ?? { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
12559
13576
  queries,
12560
13577
  };
12561
13578
  }
12562
13579
 
13580
+ function acquisitionOutstandingText(diagnostic) {
13581
+ const scopes = diagnostic?.outstandingQuota ?? [];
13582
+ if (scopes.length === 0 || scopes.some((scope) => scope.status !== "current")) {
13583
+ return "unavailable (quota projection pending)";
13584
+ }
13585
+ const units = scopes.reduce((total, scope) => ({
13586
+ core: total.core + scope.units.core,
13587
+ graphql: total.graphql + scope.units.graphql,
13588
+ }), { core: 0, graphql: 0 });
13589
+ return Number.isSafeInteger(units.core) && Number.isSafeInteger(units.graphql)
13590
+ ? `${units.core} core + ${units.graphql} GraphQL across ${scopes.length} quota scopes`
13591
+ : "unavailable (accounting overflow)";
13592
+ }
13593
+
12563
13594
  function acquisitionFailureHold(error) {
12564
13595
  if (typeof error?.providerCapability === "string") return "observer";
13596
+ if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) {
13597
+ return HTTP_PERMIT_FAILURE_HOLDS[error.coordinationReason];
13598
+ }
12565
13599
  if (SECONDARY_LIMIT_PATTERN.test(errText(error))) return "secondary";
12566
13600
  if (isRateLimited(error) || /API budget paused/i.test(errText(error))) return "primary";
12567
13601
  const attemptedHttp = Number.isSafeInteger(error?.requestMetrics?.httpRequests) &&
@@ -12889,32 +13923,58 @@ function prepareAcquisitionSnapshot(queryKey, snapshot) {
12889
13923
  }
12890
13924
 
12891
13925
  function normalizeAcquisitionStore(raw) {
12892
- if (!isRecord(raw) || raw.version !== ACQUISITION_STORE_VERSION || !validGovernorId(raw.producerEpoch) ||
13926
+ if (!isRecord(raw) || ![1, ACQUISITION_STORE_VERSION].includes(raw.version) || !validGovernorId(raw.producerEpoch) ||
12893
13927
  !isRecord(raw.subscriptions) || !isRecord(raw.queries) || !isRecord(raw.aliases)) return null;
13928
+ const legacy = raw.version === 1;
12894
13929
  const state = {
12895
13930
  version: ACQUISITION_STORE_VERSION,
12896
- producerEpoch: raw.producerEpoch,
13931
+ producerEpoch: legacy ? governorId() : raw.producerEpoch,
12897
13932
  metrics: normalizeAcquisitionMetrics(raw.metrics),
12898
- uncertainReceipts: {},
13933
+ quotaProjections: {},
13934
+ legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
12899
13935
  subscriptions: {},
12900
13936
  queries: {},
12901
13937
  aliases: {},
12902
13938
  };
12903
13939
  if (state.metrics === null) return null;
12904
- const receipts = raw.uncertainReceipts ?? {};
12905
- if (!isRecord(receipts) || Object.keys(receipts).length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return null;
12906
- for (const [id, receipt] of Object.entries(receipts)) {
12907
- const normalized = normalizeAcquisitionReceipt(receipt, id);
12908
- if (!normalized) return null;
12909
- state.uncertainReceipts[id] = normalized;
12910
- }
12911
- if (raw.uncertainReceipts !== undefined) {
12912
- const outstanding = Object.values(state.uncertainReceipts).reduce((total, receipt) => {
12913
- total[receipt.resource] += receipt.units;
12914
- return total;
12915
- }, { core: 0, graphql: 0 });
12916
- if (state.metrics.uncertainCoreUnits !== outstanding.core ||
12917
- state.metrics.uncertainGraphqlUnits !== outstanding.graphql) return null;
13940
+ if (legacy) {
13941
+ const receipts = raw.uncertainReceipts ?? {};
13942
+ if (!isRecord(receipts) || Object.keys(receipts).length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return null;
13943
+ for (const [id, receipt] of Object.entries(receipts)) {
13944
+ const normalized = normalizeAcquisitionReceipt(receipt, id);
13945
+ if (!normalized) return null;
13946
+ const key = receipt.resource === "core" ? "coreUnits" : "graphqlUnits";
13947
+ const next = state.legacyUnverified[key] + receipt.units;
13948
+ if (!Number.isSafeInteger(next)) return null;
13949
+ state.legacyUnverified[key] = next;
13950
+ state.legacyUnverified.receiptCount += 1;
13951
+ }
13952
+ if (raw.uncertainReceipts === undefined) {
13953
+ state.legacyUnverified.coreUnits = state.metrics.uncertainCoreUnits;
13954
+ state.legacyUnverified.graphqlUnits = state.metrics.uncertainGraphqlUnits;
13955
+ } else if (state.metrics.uncertainCoreUnits !== state.legacyUnverified.coreUnits ||
13956
+ state.metrics.uncertainGraphqlUnits !== state.legacyUnverified.graphqlUnits) return null;
13957
+ state.metrics.uncertainCoreUnits = 0;
13958
+ state.metrics.uncertainGraphqlUnits = 0;
13959
+ } else {
13960
+ if (!isRecord(raw.quotaProjections) || !isRecord(raw.legacyUnverified) ||
13961
+ Object.keys(raw.quotaProjections).length > ACQUISITION_MAX_QUOTA_PROJECTIONS ||
13962
+ !exactKeys(raw.legacyUnverified, ["coreUnits", "graphqlUnits", "receiptCount"]) ||
13963
+ Object.values(raw.legacyUnverified).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
13964
+ state.legacyUnverified = { ...raw.legacyUnverified };
13965
+ for (const [scopeHash, projection] of Object.entries(raw.quotaProjections)) {
13966
+ if (!/^[0-9a-f]{64}$/.test(scopeHash) || !exactKeys(projection, ["revision", "units", "at", "status", "accessKeys"]) ||
13967
+ !Number.isSafeInteger(projection.revision) || projection.revision < 0 ||
13968
+ !exactKeys(projection.units, RATE_RESOURCES) ||
13969
+ Object.values(projection.units).some((value) => !Number.isSafeInteger(value) || value < 0) ||
13970
+ !Number.isFinite(projection.at) || projection.at < 0 ||
13971
+ !["current", "unavailable"].includes(projection.status) ||
13972
+ !Array.isArray(projection.accessKeys) || projection.accessKeys.length > ACQUISITION_MAX_SUBSCRIPTIONS ||
13973
+ new Set(projection.accessKeys).size !== projection.accessKeys.length ||
13974
+ projection.accessKeys.some((key) => !/^[0-9a-f]{64}$/.test(key))) return null;
13975
+ state.quotaProjections[scopeHash] = { ...projection, units: { ...projection.units },
13976
+ accessKeys: [...projection.accessKeys] };
13977
+ }
12918
13978
  }
12919
13979
  for (const [key, value] of Object.entries(raw.aliases)) {
12920
13980
  if (typeof key !== "string" || typeof value !== "string") return null;
@@ -12933,11 +13993,15 @@ function normalizeAcquisitionStore(raw) {
12933
13993
  candidate.generation !== value.generation + 1 || !Number.isFinite(candidate.claimedAt) ||
12934
13994
  !Number.isFinite(candidate.leaseUntil) || candidate.leaseUntil < candidate.claimedAt ||
12935
13995
  typeof candidate.started !== "boolean") return null;
12936
- const hasReceiptIds = Object.hasOwn(candidate, "receiptIds");
12937
- const receiptIds = candidate.receiptIds ?? [];
12938
- if (!Array.isArray(receiptIds) || receiptIds.some((id) => typeof id !== "string" ||
12939
- !state.uncertainReceipts[id])) return null;
12940
- claim = { ...candidate, ...(hasReceiptIds ? { receiptIds } : {}) };
13996
+ if (legacy) {
13997
+ const receipts = raw.uncertainReceipts ?? {};
13998
+ const receiptIds = candidate.receiptIds ?? [];
13999
+ if (!Array.isArray(receiptIds) || receiptIds.some((id) => typeof id !== "string" || !receipts[id])) return null;
14000
+ claim = null; // v1 producer authority is fenced by the new epoch.
14001
+ } else {
14002
+ if (Object.hasOwn(candidate, "receiptIds")) return null;
14003
+ claim = { ...candidate };
14004
+ }
12941
14005
  }
12942
14006
  const snapshot = value.snapshot === null ? null
12943
14007
  : value.snapshot?.artifact
@@ -12990,37 +14054,63 @@ function loadAcquisitionMetadata(path) {
12990
14054
  }
12991
14055
  }
12992
14056
 
12993
- function readAcquisitionSnapshot(path, record) {
12994
- if (!record?.snapshot) return { ok: true, value: null };
12995
- if (Array.isArray(record.snapshot.rows)) return { ok: true, value: record.snapshot };
12996
- try {
12997
- const raw = readFileSync(join(acquisitionSnapshotRoot(path), record.snapshot.artifact), "utf8");
12998
- if (Buffer.byteLength(raw) !== record.snapshot.bytes ||
12999
- createHash("sha256").update(raw).digest("hex") !== record.snapshot.digest) {
13000
- return { ok: false, reason: "corrupt" };
14057
+ function readAcquisitionSnapshot(path, record, { deadline = performance.now() + ACQUISITION_READ_MAX_MS } = {}) {
14058
+ let current = record;
14059
+ for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
14060
+ performance.now() < deadline; attempt += 1) {
14061
+ if (!current?.snapshot) return current === record ? { ok: true, value: null }
14062
+ : { ok: false, reason: "stale" };
14063
+ if (Array.isArray(current.snapshot.rows)) {
14064
+ return { ok: true, value: current.snapshot, replaced: current !== record };
14065
+ }
14066
+ try {
14067
+ const raw = readFileSync(join(acquisitionSnapshotRoot(path), current.snapshot.artifact), "utf8");
14068
+ if (Buffer.byteLength(raw) !== current.snapshot.bytes ||
14069
+ createHash("sha256").update(raw).digest("hex") !== current.snapshot.digest) {
14070
+ return { ok: false, reason: "corrupt" };
14071
+ }
14072
+ const parsed = JSON.parse(raw);
14073
+ const snapshot = normalizeAcquisitionSnapshot(
14074
+ parsed, current.query, current.generation, parsed.producerEpoch,
14075
+ );
14076
+ return snapshot ? { ok: true, value: snapshot, replaced: current !== record }
14077
+ : { ok: false, reason: "corrupt" };
14078
+ } catch (error) {
14079
+ if (error?.code !== "ENOENT") return { ok: false, reason: "corrupt", error };
14080
+ const latest = loadAcquisitionMetadata(path);
14081
+ if (!latest.ok) return latest;
14082
+ const replacement = latest.value.queries[current.query.queryKey];
14083
+ if (!replacement) return { ok: false, reason: "stale" };
14084
+ if (replacement.snapshot?.artifact === current.snapshot.artifact) {
14085
+ return { ok: false, reason: "corrupt", error };
14086
+ }
14087
+ current = replacement;
13001
14088
  }
13002
- const parsed = JSON.parse(raw);
13003
- const snapshot = normalizeAcquisitionSnapshot(
13004
- parsed, record.query, record.generation, parsed.producerEpoch,
13005
- );
13006
- return snapshot ? { ok: true, value: snapshot } : { ok: false, reason: "corrupt" };
13007
- } catch (error) {
13008
- return { ok: false, reason: "corrupt", error };
13009
14089
  }
14090
+ return { ok: false, reason: "busy" };
13010
14091
  }
13011
14092
 
13012
- function hydrateAcquisitionStore(path, state) {
14093
+ function hydrateAcquisitionStore(path, state, deadline) {
13013
14094
  for (const record of Object.values(state.queries)) {
13014
- const loaded = readAcquisitionSnapshot(path, record);
14095
+ const loaded = readAcquisitionSnapshot(path, record, { deadline });
13015
14096
  if (!loaded.ok) return loaded;
14097
+ if (loaded.replaced) return { ok: false, reason: "stale" };
13016
14098
  record.snapshot = loaded.value;
13017
14099
  }
13018
14100
  return { ok: true, value: state };
13019
14101
  }
13020
14102
 
13021
- function loadAcquisitionStore(path) {
13022
- const loaded = loadAcquisitionMetadata(path);
13023
- return loaded.ok ? hydrateAcquisitionStore(path, loaded.value) : loaded;
14103
+ function loadAcquisitionStore(path, { onMetadataLoaded = null } = {}) {
14104
+ const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
14105
+ for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
14106
+ performance.now() < deadline; attempt += 1) {
14107
+ const loaded = loadAcquisitionMetadata(path);
14108
+ if (!loaded.ok) return loaded;
14109
+ onMetadataLoaded?.();
14110
+ const hydrated = hydrateAcquisitionStore(path, loaded.value, deadline);
14111
+ if (hydrated.reason !== "stale") return hydrated;
14112
+ }
14113
+ return { ok: false, reason: "busy" };
13024
14114
  }
13025
14115
 
13026
14116
  function writeAcquisitionArtifact(root, artifact, payload) {
@@ -13193,8 +14283,8 @@ function trimAcquisitionStore(state, protectedQueryKey = null) {
13193
14283
 
13194
14284
  async function runStartedAcquisitionTransport(markStarted, transport) {
13195
14285
  const started = await markStarted();
13196
- if (!started.ok) return started;
13197
- if (started.value?.status !== "started") return { ok: false, reason: "already-started" };
14286
+ if (!started.ok) return { ...started, notStarted: true };
14287
+ if (started.value?.status !== "started") return { ok: false, reason: "already-started", notStarted: true };
13198
14288
  return { ok: true, value: await transport() };
13199
14289
  }
13200
14290
 
@@ -13244,7 +14334,7 @@ function createAcquisitionEngine({
13244
14334
  }
13245
14335
  const loaded = loadRecordSnapshot(record);
13246
14336
  if (!loaded.ok) return loaded;
13247
- item.generation = record.generation;
14337
+ item.generation = loaded.value.generation;
13248
14338
  try { item.callback?.(loaded.value); } catch { /* subscriber callback is isolated */ }
13249
14339
  return { ok: true, value: true, snapshot: loaded.value };
13250
14340
  }
@@ -13277,7 +14367,8 @@ function createAcquisitionEngine({
13277
14367
  const record = loaded.value.queries[item.queryKey] ?? null;
13278
14368
  if (!record?.snapshot) return { ok: true, value: record };
13279
14369
  const snapshot = loadRecordSnapshot(record);
13280
- return snapshot.ok ? { ok: true, value: { ...record, snapshot: snapshot.value } } : snapshot;
14370
+ return snapshot.ok ? { ok: true, value: { ...record,
14371
+ generation: snapshot.value?.generation ?? record.generation, snapshot: snapshot.value } } : snapshot;
13281
14372
  }
13282
14373
 
13283
14374
  function heartbeat() {
@@ -13445,14 +14536,38 @@ function createAcquisitionEngine({
13445
14536
  aliasConflict: matching.length > 1 } };
13446
14537
  });
13447
14538
  if (!result.ok) return result;
13448
- const record = result.state.queries[result.value.queryKey];
13449
- const loadedSnapshot = loadRecordSnapshot(record);
13450
- if (!loadedSnapshot.ok) return loadedSnapshot;
13451
- result.value.snapshot = loadedSnapshot.value;
13452
- local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation: record?.generation ?? 0,
14539
+ // A publisher can canonicalize the query and remap this just-committed
14540
+ // shared subscription before its snapshot is hydrated. Register locally
14541
+ // first so every failure still has a cleanup owner, then follow the
14542
+ // subscription ID in fresh metadata rather than the old query key.
14543
+ local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation: 0,
13453
14544
  claimNonce: null, pending: null });
13454
14545
  ensureTimer();
13455
- return result;
14546
+ const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
14547
+ for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
14548
+ performance.now() < deadline; attempt += 1) {
14549
+ const latest = load();
14550
+ if (!latest.ok) { unsubscribe(id); return latest; }
14551
+ const shared = latest.value.subscriptions[id];
14552
+ if (!shared || shared.nonce !== nonce) { unsubscribe(id); return { ok: false, reason: "stale" }; }
14553
+ const item = local.get(id);
14554
+ item.queryKey = shared.queryKey;
14555
+ const record = latest.value.queries[shared.queryKey];
14556
+ if (!record) continue;
14557
+ const loadedSnapshot = loadRecordSnapshot(record);
14558
+ if (!loadedSnapshot.ok) {
14559
+ if (["stale", "busy"].includes(loadedSnapshot.reason)) continue;
14560
+ unsubscribe(id);
14561
+ return loadedSnapshot;
14562
+ }
14563
+ if (loadedSnapshot.replaced) continue;
14564
+ result.value.queryKey = shared.queryKey;
14565
+ result.value.snapshot = loadedSnapshot.value;
14566
+ item.generation = loadedSnapshot.value?.generation ?? record.generation;
14567
+ return result;
14568
+ }
14569
+ unsubscribe(id);
14570
+ return { ok: false, reason: "busy" };
13456
14571
  }
13457
14572
 
13458
14573
  function updateDemand(id, demandInput) {
@@ -13509,11 +14624,6 @@ function createAcquisitionEngine({
13509
14624
  const requestMetrics = produced?.requestMetrics === undefined
13510
14625
  ? {}
13511
14626
  : normalizeAcquisitionMetrics(produced.requestMetrics, { partial: true });
13512
- const receiptInputs = produced?.uncertainReceipts ?? [];
13513
- const receipts = Array.isArray(receiptInputs)
13514
- ? receiptInputs.map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
13515
- const receiptsValid = receipts && !receipts.includes(null) &&
13516
- receipts.every((receipt) => receipt.accessKey === record.query.accessKey);
13517
14627
  const loadedPrevious = loadRecordSnapshot(record);
13518
14628
  if (!loadedPrevious.ok) return loadedPrevious;
13519
14629
  if (loadedPrevious.value) record.snapshot = loadedPrevious.value;
@@ -13527,20 +14637,15 @@ function createAcquisitionEngine({
13527
14637
  snapshot = { ...snapshot, lastChangedAt: loadedPrevious.value.lastChangedAt };
13528
14638
  }
13529
14639
  if (!snapshot) {
13530
- settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics, receiptsValid ? receipts : [], now());
14640
+ settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics, [], now());
13531
14641
  return { value: { ok: false, reason: "invalid" } };
13532
14642
  }
13533
14643
  const previous = record.snapshot;
13534
14644
  const previousClaim = record.claim;
13535
- if (requestMetrics === null || !receiptsValid) {
14645
+ if (requestMetrics === null) {
13536
14646
  settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics ?? {}, [], now());
13537
14647
  return { value: { ok: false, reason: "invalid" } };
13538
14648
  }
13539
- const receiptIds = new Set(Object.keys(state.uncertainReceipts ?? {}));
13540
- const novelReceipts = receipts.filter((receipt) => !receiptIds.has(receipt.id));
13541
- if (receiptIds.size + novelReceipts.length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) {
13542
- return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
13543
- }
13544
14649
  record.generation = effectiveClaim.generation;
13545
14650
  record.snapshot = snapshot;
13546
14651
  record.claim = null;
@@ -13552,14 +14657,7 @@ function createAcquisitionEngine({
13552
14657
  record.claim = previousClaim;
13553
14658
  return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
13554
14659
  }
13555
- if (Object.hasOwn(requestMetrics, "uncertainCoreUnits") ||
13556
- Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
13557
- releaseKnownAcquisitionUncertainty(state, { ...record, claim: previousClaim }, requestMetrics);
13558
- } else {
13559
- releaseAcquisitionUncertainty(state, { ...record, claim: previousClaim });
13560
- }
13561
14660
  addAcquisitionRequestMetrics(state, requestMetrics);
13562
- for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
13563
14661
  settleAcquisitionWaiters(state, record.query.queryKey, effectiveClaim.generation, now());
13564
14662
  return { value: { ok: true, value: { role: "producer", snapshot,
13565
14663
  queryKey: record.query.queryKey, remapped } } };
@@ -13613,10 +14711,8 @@ function createAcquisitionEngine({
13613
14711
  function settleFailure(id, claim, failure) {
13614
14712
  const item = local.get(id);
13615
14713
  const requestMetrics = normalizeAcquisitionMetrics(failure?.requestMetrics ?? {}, { partial: true });
13616
- const receipts = Array.isArray(failure?.uncertainReceipts ?? [])
13617
- ? (failure?.uncertainReceipts ?? []).map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
13618
14714
  const hold = failure?.hold ?? "primary";
13619
- if (!item || !isRecord(claim) || requestMetrics === null || !receipts || receipts.includes(null) ||
14715
+ if (!item || !isRecord(claim) || requestMetrics === null ||
13620
14716
  !ACQUISITION_HOLD_REASONS.has(hold)) {
13621
14717
  return { ok: false, reason: "invalid" };
13622
14718
  }
@@ -13625,13 +14721,7 @@ function createAcquisitionEngine({
13625
14721
  if (!record || record.claim?.nonce !== claim.nonce || record.claim.generation !== claim.generation) {
13626
14722
  return { value: { ok: false, reason: "stale", definitive: true } };
13627
14723
  }
13628
- if (receipts.some((receipt) => receipt.accessKey !== record.query.accessKey) ||
13629
- Object.keys(state.uncertainReceipts ?? {}).length +
13630
- receipts.filter((receipt) => !state.uncertainReceipts?.[receipt.id]).length >
13631
- ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return { ok: false, reason: "capacity" };
13632
14724
  addAcquisitionRequestMetrics(state, requestMetrics);
13633
- releaseKnownAcquisitionUncertainty(state, record, requestMetrics);
13634
- for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
13635
14725
  settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, now());
13636
14726
  record.claim = null;
13637
14727
  record.hold = { reason: hold, at: now() };
@@ -13654,7 +14744,11 @@ function createAcquisitionEngine({
13654
14744
  const item = local.get(id);
13655
14745
  if (!item || reason !== null && !ACQUISITION_HOLD_REASONS.has(reason) ||
13656
14746
  expected !== null && (!isRecord(expected) || !TAB_KEYS.includes(expected.resource) ||
13657
- typeof expected.accessKey !== "string" || expected.accessKey.length === 0)) {
14747
+ typeof expected.accessKey !== "string" || expected.accessKey.length === 0 ||
14748
+ expected.currentHold !== undefined &&
14749
+ (!isRecord(expected.currentHold) ||
14750
+ !ACQUISITION_HOLD_REASONS.has(expected.currentHold.reason) ||
14751
+ !Number.isFinite(expected.currentHold.at)))) {
13658
14752
  return { ok: false, reason: "invalid" };
13659
14753
  }
13660
14754
  return transact((state) => {
@@ -13662,6 +14756,11 @@ function createAcquisitionEngine({
13662
14756
  if (!record) return { ok: false, reason: "stale" };
13663
14757
  if (expected && (record.query.resource !== expected.resource ||
13664
14758
  record.query.accessKey !== expected.accessKey)) return { ok: false, reason: "stale-access" };
14759
+ if (expected?.currentHold !== undefined &&
14760
+ (record.hold?.reason !== expected.currentHold.reason ||
14761
+ record.hold?.at !== expected.currentHold.at)) {
14762
+ return { ok: false, reason: "stale-hold" };
14763
+ }
13665
14764
  if ((record.hold?.reason ?? null) === reason) return { changed: false, value: true };
13666
14765
  const next = reason === null ? null : { reason, at: now() };
13667
14766
  record.hold = next;
@@ -13693,28 +14792,10 @@ function createAcquisitionEngine({
13693
14792
  if (now() >= acquisitionClaimDeadline(record.claim)) {
13694
14793
  return { ok: false, reason: "stale" };
13695
14794
  }
13696
- const receiptIds = [];
13697
- for (const resource of RATE_RESOURCES) {
13698
- if (costs[resource] <= 0) continue;
13699
- const epoch = typeof supplied?.epochs?.[resource] === "string" && supplied.epochs[resource].length > 0
13700
- ? supplied.epochs[resource] : "unknown";
13701
- const receipt = {
13702
- id: `${reservationId}:${resource}`,
13703
- reservationId,
13704
- accessKey: record.query.accessKey,
13705
- resource,
13706
- epoch,
13707
- units: costs[resource],
13708
- startedAt: now(),
13709
- };
13710
- if (!addAcquisitionReceipt(state, receipt)) return { ok: false, reason: "capacity" };
13711
- receiptIds.push(receipt.id);
13712
- }
13713
14795
  record.claim.started = true;
13714
14796
  record.claim.startedAt = now();
13715
14797
  record.claim.reservationId = reservationId;
13716
14798
  record.claim.startReceipt = startReceipt;
13717
- record.claim.receiptIds = receiptIds;
13718
14799
  return { value: { status: "started", reservationId } };
13719
14800
  });
13720
14801
  }
@@ -13764,7 +14845,8 @@ function createAcquisitionEngine({
13764
14845
  const claimed = transact((state) => {
13765
14846
  const reaped = reapAcquisitionSubscriptions(state, now(), kill);
13766
14847
  const subscription = state.subscriptions[id];
13767
- const record = state.queries[item.queryKey];
14848
+ const queryKey = subscription?.queryKey;
14849
+ const record = state.queries[queryKey];
13768
14850
  if (!subscription || subscription.nonce !== item.nonce || !record) return { ok: false, reason: "stale" };
13769
14851
  const requestedGeneration = force
13770
14852
  ? Math.max(subscription.requestedGeneration,
@@ -13776,8 +14858,8 @@ function createAcquisitionEngine({
13776
14858
  }
13777
14859
  if (requestedGeneration <= record.generation && record.snapshot?.nextDueAt > now()) {
13778
14860
  return { changed: reaped > 0 || demandChanged,
13779
- value: { role: "follower", reason: "fresh", snapshot: record.snapshot,
13780
- sharingCount: acquisitionSharingCount(state, item.queryKey) } };
14861
+ value: { role: "follower", reason: "fresh", queryKey, snapshot: record.snapshot,
14862
+ sharingCount: acquisitionSharingCount(state, queryKey) } };
13781
14863
  }
13782
14864
  if (record.claim) {
13783
14865
  const absoluteDeadline = acquisitionClaimDeadline(record.claim);
@@ -13792,10 +14874,10 @@ function createAcquisitionEngine({
13792
14874
  }
13793
14875
  return { changed: reaped > 0 || demandChanged || joined,
13794
14876
  value: { role: "follower", reason: expired ? `owner-${owner}` : "claimed",
13795
- snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state, item.queryKey) } };
14877
+ queryKey, snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state, queryKey) } };
13796
14878
  }
13797
14879
  }
13798
- const nonce = createId("acquisition-claim", item.queryKey, record.generation + 1);
14880
+ const nonce = createId("acquisition-claim", queryKey, record.generation + 1);
13799
14881
  record.claim = {
13800
14882
  pid,
13801
14883
  nonce,
@@ -13809,11 +14891,13 @@ function createAcquisitionEngine({
13809
14891
  subscription.waitingSinceAt = null;
13810
14892
  record.lastUsedAt = now();
13811
14893
  return { value: { role: "producer", nonce, generation: record.claim.generation,
13812
- query: record.query, snapshot: record.snapshot, demand: aggregateAcquisitionDemand(state, item.queryKey),
13813
- sharingCount: acquisitionSharingCount(state, item.queryKey) } };
14894
+ queryKey, query: record.query, snapshot: record.snapshot,
14895
+ demand: aggregateAcquisitionDemand(state, queryKey),
14896
+ sharingCount: acquisitionSharingCount(state, queryKey) } };
13814
14897
  });
14898
+ if (claimed.ok && claimed.value?.queryKey) item.queryKey = claimed.value.queryKey;
13815
14899
  if (claimed.ok && claimed.value?.snapshot) {
13816
- const record = claimed.state.queries[item.queryKey];
14900
+ const record = claimed.state.queries[claimed.value.queryKey];
13817
14901
  const loadedSnapshot = loadRecordSnapshot(record);
13818
14902
  if (!loadedSnapshot.ok) return loadedSnapshot;
13819
14903
  claimed.value.snapshot = loadedSnapshot.value;
@@ -13921,21 +15005,58 @@ function createAcquisitionEngine({
13921
15005
  });
13922
15006
  }
13923
15007
 
13924
- function reconcileUncertainty(accessKey, evidence) {
13925
- if (typeof accessKey !== "string" || accessKey.length === 0 || !isRecord(evidence)) {
15008
+ function syncQuotaProjection(scope) {
15009
+ if (!isRecord(scope) || !/^[0-9a-f]{64}$/.test(scope.hash ?? "")) {
13926
15010
  return { ok: false, reason: "invalid" };
13927
15011
  }
15012
+ // Quota is read before the acquisition lock. This diagnostic copy never
15013
+ // grants requests and a failed read cannot block source publication.
15014
+ const observed = inspectGovernor(scope, now());
15015
+ if (!observed.ok) {
15016
+ return transact((state) => {
15017
+ const previous = state.quotaProjections[scope.hash];
15018
+ if (!previous || previous.status === "unavailable") return { changed: false, value: false };
15019
+ previous.status = "unavailable";
15020
+ return { value: false };
15021
+ });
15022
+ }
15023
+ const ledger = observed.value;
15024
+ const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
15025
+ Object.values(ledger.reservations).reduce((sum, reservation) =>
15026
+ sum + reservationCost(reservation, resource, ledger.leases, now()),
15027
+ ledger.debt[resource].unresolvedUnits + ledger.debt[resource].quiescentUnits +
15028
+ reservationCost(ledger.controlReceipts[resource], resource, ledger.leases, now()))]));
15029
+ if (Object.values(units).some((value) => !Number.isSafeInteger(value) || value < 0)) {
15030
+ return { ok: false, reason: "accounting-overflow" };
15031
+ }
13928
15032
  return transact((state) => {
13929
- let changed = false;
13930
- for (const [id, receipt] of Object.entries(state.uncertainReceipts ?? {})) {
13931
- const observed = evidence[receipt.resource];
13932
- if (receipt.accessKey !== accessKey || !isRecord(observed) ||
13933
- typeof observed.epoch !== "string" || !Number.isFinite(observed.observedAt)) continue;
13934
- const reset = receipt.epoch !== "unknown" && observed.epoch !== receipt.epoch;
13935
- if (!reset && observed.observedAt < receipt.startedAt) continue;
13936
- changed = removeAcquisitionReceipt(state, id) || changed;
15033
+ const activeKeys = new Set(Object.values(state.subscriptions)
15034
+ .map((subscription) => state.queries[subscription.queryKey]?.query.accessKey)
15035
+ .filter(Boolean));
15036
+ const currentAccessKey = /^[0-9a-f]{64}$/.test(scope.accessKey ?? "")
15037
+ ? scope.accessKey : null;
15038
+ if (currentAccessKey) {
15039
+ for (const [hash, projection] of Object.entries(state.quotaProjections)) {
15040
+ if (hash !== scope.hash) projection.accessKeys = projection.accessKeys.filter((key) =>
15041
+ key !== currentAccessKey);
15042
+ }
13937
15043
  }
13938
- return { changed, value: { ...state.metrics } };
15044
+ const previous = state.quotaProjections[scope.hash];
15045
+ if (previous && previous.revision > ledger.revision) return { changed: false, value: false };
15046
+ if (!previous && Object.keys(state.quotaProjections).length >= ACQUISITION_MAX_QUOTA_PROJECTIONS) {
15047
+ const oldest = Object.entries(state.quotaProjections)
15048
+ .filter(([, projection]) => !projection.accessKeys.some((key) => activeKeys.has(key)))
15049
+ .sort((a, b) => a[1].at - b[1].at)[0];
15050
+ if (!oldest) return { changed: false, value: false };
15051
+ delete state.quotaProjections[oldest[0]];
15052
+ }
15053
+ const accessKeys = previous?.accessKeys.filter((key) => activeKeys.has(key)) ?? [];
15054
+ if (currentAccessKey && !accessKeys.includes(currentAccessKey) &&
15055
+ accessKeys.length < ACQUISITION_MAX_SUBSCRIPTIONS) accessKeys.push(currentAccessKey);
15056
+ state.quotaProjections[scope.hash] = {
15057
+ revision: ledger.revision, units, at: now(), status: "current", accessKeys,
15058
+ };
15059
+ return { value: true };
13939
15060
  });
13940
15061
  }
13941
15062
 
@@ -13948,7 +15069,7 @@ function createAcquisitionEngine({
13948
15069
  inspect,
13949
15070
  diagnostics,
13950
15071
  recordMetrics,
13951
- reconcileUncertainty,
15072
+ syncQuotaProjection,
13952
15073
  setHold,
13953
15074
  close,
13954
15075
  path,
@@ -13964,6 +15085,27 @@ function setRuntimeAcquisitionHold(engine, subscriptions, resource, reason, acce
13964
15085
  return engine.setHold(subscription.id, reason, { resource, accessKey });
13965
15086
  }
13966
15087
 
15088
+ function clearScheduledRuntimeControlHold(engine, subscriptions, resource, accessKey, observedAt) {
15089
+ const subscription = subscriptions?.get(resource);
15090
+ if (!subscription?.id) return { ok: false, reason: "stale" };
15091
+ if (typeof accessKey !== "string" || !Number.isFinite(observedAt)) {
15092
+ return { ok: false, reason: "stale-access" };
15093
+ }
15094
+ const inspected = engine.inspect(subscription.id);
15095
+ if (!inspected.ok) return inspected;
15096
+ const record = inspected.value;
15097
+ if (record?.query?.resource !== resource || record.query.accessKey !== accessKey) {
15098
+ return { ok: false, reason: "stale-access" };
15099
+ }
15100
+ const hold = record.hold;
15101
+ if (!hold) return { ok: true, value: "already-clear" };
15102
+ if (hold.reason !== "coordination") return { ok: true, value: "retained" };
15103
+ if (hold.at >= observedAt) return { ok: false, reason: "newer-hold" };
15104
+ const cleared = engine.setHold(subscription.id, null,
15105
+ { resource, accessKey, currentHold: hold });
15106
+ return cleared.ok ? { ok: true, value: "cleared" } : cleared;
15107
+ }
15108
+
13967
15109
  // The one place the cadence table is decided. It used to be spread across a
13968
15110
  // fixed active floor, a fixed four-floor background slot, and a Security-only
13969
15111
  // unchanged rule -- three policies that could disagree, and did: Security
@@ -13977,7 +15119,10 @@ function pollPolicyInterval({
13977
15119
  unchangedCount = 0,
13978
15120
  inProgressCI = false,
13979
15121
  background = "all",
15122
+ requested = false,
13980
15123
  }) {
15124
+ if (background === "on-demand" &&
15125
+ (demand !== "active" || (tab !== "actions" && !requested))) return Number.POSITIVE_INFINITY;
13981
15126
  if (demand === "none") return Number.POSITIVE_INFINITY;
13982
15127
  if (demand === "inactive") {
13983
15128
  if (background === "off") return Number.POSITIVE_INFINITY;
@@ -14105,6 +15250,7 @@ function pollSchedule({
14105
15250
  demand: isActive ? "active" : "inactive",
14106
15251
  unchangedCount: states[key]?.unchangedCount ?? 0,
14107
15252
  inProgressCI: states[key]?.inProgressCI === true,
15253
+ requested: states[key]?.requested === true,
14108
15254
  background,
14109
15255
  });
14110
15256
  if (!Number.isFinite(step)) {
@@ -14249,7 +15395,7 @@ function governorDataReady(refreshResult, snapshot, activeKey, nowMs) {
14249
15395
  return RATE_RESOURCES.every((resource) => {
14250
15396
  if (costs[resource] <= 0) return true;
14251
15397
  const budget = snapshot.value.budgets[resource];
14252
- const available = availableForGrant({ budget, resource, nowMs });
15398
+ const available = governorAvailableForGrant(snapshot.value, resource, nowMs);
14253
15399
  return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource) &&
14254
15400
  budget.blockUntil <= nowMs && available.mode === "open" &&
14255
15401
  available.spendable >= costs[resource];
@@ -14456,6 +15602,333 @@ function coordinationNotice(reason) {
14456
15602
  return "Can't coordinate API use — retrying";
14457
15603
  }
14458
15604
 
15605
+ // Recovery diagnostics have no request payloads, credential identifiers or
15606
+ // owner capabilities. The journal is advisory: a failed write must never deny
15607
+ // an otherwise valid data operation. The quota ledger remains authoritative.
15608
+ const RECOVERY_JOURNAL_MAX_ENTRIES = 128;
15609
+ const RECOVERY_JOURNAL_MAX_BYTES = 128 * 1024;
15610
+ const RECOVERY_ENTRY_MAX_BYTES = 1024;
15611
+ const RECOVERY_COPY = Object.freeze({
15612
+ busy: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
15613
+ stale: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
15614
+ "identity-busy": ["GitHub budget check is shared", "Retry automatically after the current check", "Budget check; retry"],
15615
+ "transport-busy": ["Shared GitHub request is in progress", "Retry automatically", "Request; retry"],
15616
+ unwritable: ["Local coordination cannot be written", "Run gh-glance --doctor; check storage permissions", "Store blocked; doctor"],
15617
+ corrupt: ["Local coordination is corrupt", "Run gh-glance --doctor; preserve the reported state file", "State corrupt; doctor"],
15618
+ capacity: ["Local coordination is full", "Run gh-glance --doctor; preserve the reported state file", "State full; doctor"],
15619
+ "accounting-overflow": ["Local coordination accounting overflow", "Run gh-glance --doctor; preserve the reported state file", "Overflow; doctor"],
15620
+ "reservations-invalid": ["Local coordination reservation invalid", "Run gh-glance --doctor; preserve the reported state file", "Reservation; doctor"],
15621
+ "external-factor-invalid": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
15622
+ "pacing-invalid": ["Local coordination pacing invalid", "Run gh-glance --doctor; preserve the reported state file", "Pacing invalid; doctor"],
15623
+ "budget-resource": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
15624
+ "legacy-unresolved": ["Older requests still reserve quota", "Verify old children ended; a verified system restart can resolve unknown ownership", "Old debt; restart"],
15625
+ "legacy-corrupt": ["Older coordination evidence is corrupt", "Run gh-glance --doctor; preserve the reported state file", "Old state; doctor"],
15626
+ "migration-hold": ["Older session still uses coordination", "Wait for the older quota reset; retry automatically at the shown time", "Old quota; wait"],
15627
+ "restart-required": ["Older session still uses coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
15628
+ "identity-backoff": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
15629
+ "identity-capacity": ["GitHub budget check allowance is full", "Retry automatically when the rolling window renews", "Budget check; retry"],
15630
+ "budget-unknown": ["GitHub budget check pending", "Retry automatically", "Budget check; retry"],
15631
+ "budget-stale": ["GitHub budget evidence is old", "Retry the shared budget check automatically", "Budget old; retry"],
15632
+ "budget-future": ["Clock changed; budget needs recheck", "Retry the shared budget check automatically", "Clock; retry"],
15633
+ "probe-failed": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
15634
+ "block-unpublished": ["Saving GitHub request limit", "Retry sharing the hold automatically", "Saving hold; retry"],
15635
+ "budget-reset": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
15636
+ "rate-limit": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
15637
+ "rate-limited": ["GitHub request limit", "Retry automatically after the shared hold", "Limit; retry"],
15638
+ "local-reserve": ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
15639
+ reserve: ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
15640
+ blocked: ["GitHub request hold", "Retry automatically when the hold ends", "Request hold; retry"],
15641
+ secondary: ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
15642
+ "secondary-rate-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
15643
+ "abuse-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
15644
+ "throttle-paused": ["Repeated GitHub request limit", "Press r to retry after checking the hold", "Limit; press r"],
15645
+ "network-outage": ["GitHub connection failed", "Retry automatically when the connection returns", "Network; retry"],
15646
+ "credential-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
15647
+ "identity-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
15648
+ disconnected: ["Collector connection unavailable", "Restore the selected collector connection", "Collector; connection"],
15649
+ "app-auth": ["GitHub App authorization unavailable", "Restore the selected App authorization", "App; restore auth"],
15650
+ "security-incomplete": ["Security data incomplete", "Run gh-glance --doctor --probe for endpoint access", "Security; doctor"],
15651
+ "receipt-retry": ["Saving request result", "Retry automatically without repeating the request", "Saving; retry"],
15652
+ "interrupted-request": ["Recovering interrupted request", "Retry source observation when quota allows", "Interrupted; retry"],
15653
+ "request-queue": ["Recovering local request queue", "Retry automatically after compaction", "Queue; retry"],
15654
+ "acquisition-busy": ["Acquisition store is busy", "Check the shared lock; retry automatically", "Store lock; retry"],
15655
+ "clock-recovery": ["Rechecking after sleep or clock change", "Retry source observation automatically", "Clock; retry"],
15656
+ "auth-problem": ["GitHub authorization failed", "Run gh auth status, then restore authorization", "Auth; gh auth status"],
15657
+ unavailable: ["Repository or endpoint unavailable", "Check the selected repository and permissions", "Repo; check access"],
15658
+ "source-incomplete": ["GitHub response incomplete", "Retry source observation automatically", "Incomplete; retry"],
15659
+ "unknown-scope": ["GitHub account scope unavailable", "Run gh auth status; retry when identity is verified", "Login; gh auth status"],
15660
+ "version-mismatch": ["Older session cannot share coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
15661
+ "unsafe-permissions": ["Local coordination permissions are unsafe", "Run gh-glance --doctor; repair directory permissions", "Permissions; doctor"],
15662
+ "disk-full": ["Local coordination storage is full", "Run gh-glance --doctor; free local storage", "Disk full; doctor"],
15663
+ coordination: ["Local coordination unavailable", "Run gh-glance --doctor; retry when storage works", "Coordination; doctor"],
15664
+ });
15665
+
15666
+ function recoveryCause({ reason, resource = null, at = Date.now(), retryAt = null,
15667
+ sourceAt = null, receiptUnits = null, debtUnits = null, oldestUnresolvedAt = null,
15668
+ observerAt = null, terminalizationBacklog = null, lastFailedTransition = null,
15669
+ origin = "coordination", tab = null } = {}) {
15670
+ const code = typeof reason === "string" && Object.hasOwn(RECOVERY_COPY, reason)
15671
+ ? reason : "coordination";
15672
+ const copy = RECOVERY_COPY[code] ?? RECOVERY_COPY.coordination;
15673
+ const time = (value) => Number.isFinite(value) && value >= 0 ? value : null;
15674
+ const units = (value) => Number.isSafeInteger(value) && value >= 0 ? value : null;
15675
+ return {
15676
+ code, resource: RATE_RESOURCES.includes(resource) ? resource : null,
15677
+ origin: origin === "source" ? "source" : "coordination",
15678
+ tab: TAB_KEYS.includes(tab) ? tab : null,
15679
+ at: time(at) ?? Date.now(), firstAt: time(at) ?? Date.now(),
15680
+ retryAt: time(retryAt), sourceAt: time(sourceAt),
15681
+ receiptUnits: units(receiptUnits), debtUnits: units(debtUnits),
15682
+ oldestUnresolvedAt: time(oldestUnresolvedAt), observerAt: time(observerAt),
15683
+ terminalizationBacklog: units(terminalizationBacklog),
15684
+ lastFailedTransition: typeof lastFailedTransition === "string" && Object.hasOwn(RECOVERY_COPY, lastFailedTransition)
15685
+ ? lastFailedTransition : code,
15686
+ summary: copy[0], action: copy[1], short: copy[2],
15687
+ };
15688
+ }
15689
+
15690
+ function shouldClearRecoveryCause(cause, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
15691
+ if (!cause) return false;
15692
+ if (scheduledHold !== null) {
15693
+ return scheduledHold.ok === true && scheduledHold.value === "cleared" &&
15694
+ cause.origin === "coordination" && ["busy", "stale", "acquisition-busy"].includes(cause.code);
15695
+ }
15696
+ return cause.origin !== "source" || sourceSuccess && cause.tab === tab;
15697
+ }
15698
+
15699
+ function recoveryRetryAt(decision, budget, observer, nowMs) {
15700
+ if (budget?.blockUntil > nowMs) return budget.blockUntil;
15701
+ if (["reservations-invalid", "external-factor-invalid", "pacing-invalid", "budget-resource",
15702
+ "accounting-overflow", "corrupt", "unwritable", "capacity"].includes(decision?.reason)) return null;
15703
+ const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision?.reason);
15704
+ if (decision?.mode === "probe" && !resetHold) {
15705
+ return observer?.nextAt > nowMs ? observer.nextAt : null;
15706
+ }
15707
+ const deadline = decision?.retryAt ?? decision?.resetMs ?? budget?.resetMs;
15708
+ return deadline > nowMs ? deadline : null;
15709
+ }
15710
+
15711
+ function terminalizationRecoveryCode(terminal) {
15712
+ if (terminal?.status === "retryable") return "receipt-retry";
15713
+ if (terminal?.status === "compacted") return "interrupted-request";
15714
+ if (terminal?.reason === "completion-capacity") return "capacity";
15715
+ return terminal?.reason ?? terminal?.status ?? "coordination";
15716
+ }
15717
+
15718
+ function requestFailureRecoveryCode(error, verdict) {
15719
+ if (verdict === "rate-limited") return "rate-limited";
15720
+ if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) return error.coordinationReason;
15721
+ if (verdict === "other" && error?.httpStarted === true) return "interrupted-request";
15722
+ return verdict === "other" ? "network-outage" : verdict;
15723
+ }
15724
+
15725
+ function requestFailureRecovery(error, verdict) {
15726
+ const reason = requestFailureRecoveryCode(error, verdict);
15727
+ return { reason, origin: reason === "transport-busy" ? "coordination" : "source" };
15728
+ }
15729
+
15730
+ function wallClockRecovery(previousAt, at, intervalMs) {
15731
+ if (!Number.isFinite(previousAt) || !Number.isFinite(at)) return null;
15732
+ return at < previousAt || at - previousAt > Math.max(30_000, 3 * intervalMs)
15733
+ ? "clock-recovery" : null;
15734
+ }
15735
+
15736
+ function collectorSourceAdvanced(previousAt, sourceAt) {
15737
+ return Number.isFinite(previousAt) && Number.isFinite(sourceAt) && sourceAt > previousAt;
15738
+ }
15739
+
15740
+ function cleanupQueueForResource(queue, resource) {
15741
+ let count = 0;
15742
+ for (const item of queue) if (tabRequestCost(item.cleanupKey)?.[resource] > 0) count += 1;
15743
+ return count;
15744
+ }
15745
+
15746
+ function presentRecovery(cause, { nowMs = Date.now(), cols = 80 } = {}) {
15747
+ if (!cause) return [];
15748
+ const width = Math.max(1, Number.isSafeInteger(cols) ? cols - 1 : 79);
15749
+ const resource = cause.resource === "graphql" ? "GraphQL" : cause.resource === "core" ? "Core" : "GitHub";
15750
+ const age = Number.isFinite(cause.sourceAt) && cause.sourceAt <= nowMs
15751
+ ? `cached ${formatDuration(nowMs - cause.sourceAt)}` : "cached age unavailable";
15752
+ const retry = Number.isFinite(cause.retryAt)
15753
+ ? `; next ${statusInterval(cause.retryAt, nowMs)}` : "";
15754
+ const summary = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
15755
+ ? `Older requests still reserve ${cause.debtUnits} units` : cause.summary;
15756
+ if (width < 60) {
15757
+ const action = cause.action.toLowerCase();
15758
+ const narrowAction = cause.code === "legacy-unresolved" ? "Check children; restart"
15759
+ : action.includes("gh-glance --doctor") ? "Run gh-glance --doctor"
15760
+ : action.includes("gh auth") ? "Run gh auth status"
15761
+ : action.includes("selected repository") ? "Check repository access"
15762
+ : action.includes("connection") ? "Restore connection"
15763
+ : action.includes("app authorization") ? "Restore App auth"
15764
+ : action.includes("permissions") ? "Check permissions"
15765
+ : action.includes("restart") ? "Restart and recheck"
15766
+ : action.includes("press r") ? "Press r to retry"
15767
+ : "Retry automatically";
15768
+ const narrowDeadline = Number.isFinite(cause.retryAt)
15769
+ ? statusInterval(cause.retryAt, nowMs) : null;
15770
+ const timedAction = narrowDeadline && cause.code === "migration-hold"
15771
+ ? `Wait; next ${narrowDeadline}`
15772
+ : narrowDeadline && narrowAction === "Retry automatically"
15773
+ ? `Retry; next ${narrowDeadline}`
15774
+ : narrowDeadline && narrowAction === "Press r to retry"
15775
+ ? `Press r after ${narrowDeadline}` : narrowAction;
15776
+ const short = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
15777
+ ? `Debt ${cause.debtUnits} units` : cause.short.split(";")[0];
15778
+ return [`${resource}: ${short}`.slice(0, width), timedAction.slice(0, width), age.slice(0, width)];
15779
+ }
15780
+ const full = `${resource}: ${summary}; ${cause.action}${retry}; ${age}`;
15781
+ if (full.length <= width) return [full];
15782
+ return [`${resource}: ${summary}`.slice(0, width),
15783
+ `${cause.action}${retry}`.slice(0, width), age.slice(0, width)];
15784
+ }
15785
+
15786
+ function recoveryJournalPath(scope) {
15787
+ return `${scope.path}.recovery.json`;
15788
+ }
15789
+
15790
+ function emptyRecoveryJournal() {
15791
+ return { version: 1, current: { core: null, graphql: null }, entries: [] };
15792
+ }
15793
+
15794
+ const RECOVERY_EVENT_KEYS = ["code", "resource", "at", "firstAt", "retryAt", "sourceAt",
15795
+ "receiptUnits", "debtUnits", "oldestUnresolvedAt", "observerAt", "terminalizationBacklog",
15796
+ "lastFailedTransition", "origin", "tab"];
15797
+
15798
+ function recoveryEvent(entry) {
15799
+ if (!isRecord(entry)) return null;
15800
+ if (exactKeys(entry, ["code", "resource", "at"]) && entry.code === "clear" &&
15801
+ RATE_RESOURCES.includes(entry.resource) && Number.isFinite(entry.at)) return entry;
15802
+ // An earlier clock sample can arrive after a later one for the same cause.
15803
+ // Repair only this bounded timestamp skew; every other field stays subject
15804
+ // to the strict journal validation below.
15805
+ const event = Number.isFinite(entry.at) && Number.isFinite(entry.firstAt) && entry.firstAt > entry.at
15806
+ ? { ...entry, firstAt: entry.at } : entry;
15807
+ if (!exactKeys(event, RECOVERY_EVENT_KEYS) || !RATE_RESOURCES.includes(event.resource) ||
15808
+ typeof event.code !== "string" || !Object.hasOwn(RECOVERY_COPY, event.code) ||
15809
+ !Number.isFinite(event.at) || !Number.isFinite(event.firstAt) ||
15810
+ ["retryAt", "sourceAt", "oldestUnresolvedAt", "observerAt"].some((key) =>
15811
+ event[key] !== null && (!Number.isFinite(event[key]) || event[key] < 0)) ||
15812
+ ["receiptUnits", "debtUnits", "terminalizationBacklog"].some((key) =>
15813
+ event[key] !== null && (!Number.isSafeInteger(event[key]) || event[key] < 0)) ||
15814
+ typeof event.lastFailedTransition !== "string" ||
15815
+ !Object.hasOwn(RECOVERY_COPY, event.lastFailedTransition) ||
15816
+ !["source", "coordination"].includes(event.origin) ||
15817
+ (event.tab !== null && !TAB_KEYS.includes(event.tab)) ||
15818
+ Buffer.byteLength(JSON.stringify(event)) > RECOVERY_ENTRY_MAX_BYTES) return null;
15819
+ return event;
15820
+ }
15821
+
15822
+ function visibleRecoveryEvent(entry) {
15823
+ if (!entry || entry.code === "clear") return entry;
15824
+ const copy = RECOVERY_COPY[entry.code] ?? RECOVERY_COPY.coordination;
15825
+ return { ...entry, summary: copy[0], action: copy[1], short: copy[2] };
15826
+ }
15827
+
15828
+ function readRecoveryTransitions(scope) {
15829
+ const path = recoveryJournalPath(scope);
15830
+ let raw;
15831
+ try {
15832
+ const file = lstatSync(path);
15833
+ if (!file.isFile() || file.size > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
15834
+ raw = readFileSync(path, "utf8");
15835
+ if (Buffer.byteLength(raw) > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
15836
+ } catch (error) {
15837
+ return { ...emptyRecoveryJournal(), status: error?.code === "ENOENT" ? "missing" : "unavailable" };
15838
+ }
15839
+ try {
15840
+ const journal = JSON.parse(raw);
15841
+ const entries = journal.entries?.map(recoveryEvent);
15842
+ const current = journal.current && Object.fromEntries(RATE_RESOURCES.map((resource) => [
15843
+ resource, journal.current[resource] === null ? null : recoveryEvent(journal.current[resource]),
15844
+ ]));
15845
+ if (!exactKeys(journal, ["version", "current", "entries"]) || journal.version !== 1 ||
15846
+ !exactKeys(journal.current, RATE_RESOURCES) || !Array.isArray(journal.entries) ||
15847
+ journal.entries.length > RECOVERY_JOURNAL_MAX_ENTRIES ||
15848
+ entries.some((entry) => entry === null) ||
15849
+ RATE_RESOURCES.some((resource) => journal.current[resource] !== null &&
15850
+ (current[resource] === null || current[resource].code === "clear"))) {
15851
+ return { ...emptyRecoveryJournal(), status: "corrupt" };
15852
+ }
15853
+ return { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
15854
+ [resource, visibleRecoveryEvent(current[resource])])),
15855
+ entries: entries.map(visibleRecoveryEvent), status: "healthy" };
15856
+ } catch {
15857
+ return { ...emptyRecoveryJournal(), status: "corrupt" };
15858
+ }
15859
+ }
15860
+
15861
+ function recordRecoveryTransition(scope, event) {
15862
+ if (!scope?.path || !RATE_RESOURCES.includes(event?.resource)) return { ok: false, reason: "invalid" };
15863
+ const path = recoveryJournalPath(scope);
15864
+ return withPersistenceLock(path, () => {
15865
+ const previous = readRecoveryTransitions(scope);
15866
+ if (!["healthy", "missing"].includes(previous.status)) return { ok: false, reason: previous.status };
15867
+ const plain = (entry) => entry && Object.fromEntries(RECOVERY_EVENT_KEYS.map((key) => [key, entry[key]]));
15868
+ const journal = { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
15869
+ [resource, plain(previous.current[resource])])),
15870
+ entries: previous.entries.map((entry) => entry.code === "clear" ? entry : plain(entry)) };
15871
+ const former = journal.current[event.resource];
15872
+ const at = Number.isFinite(event.at) ? event.at : Date.now();
15873
+ if (event.clear === true) {
15874
+ if (!former) return { ok: true, changed: false };
15875
+ journal.current[event.resource] = null;
15876
+ journal.entries.push({ code: "clear", resource: event.resource, at });
15877
+ } else {
15878
+ const entry = plain(recoveryCause({ ...event, reason: event.reason ?? event.code, at }));
15879
+ entry.firstAt = former?.code === entry.code && former?.origin === entry.origin &&
15880
+ former?.tab === entry.tab ? Math.min(former.firstAt, at) : at;
15881
+ if (Buffer.byteLength(JSON.stringify(entry)) > RECOVERY_ENTRY_MAX_BYTES) return { ok: false, reason: "capacity" };
15882
+ journal.current[event.resource] = entry;
15883
+ if (former?.code !== entry.code || former?.origin !== entry.origin || former?.tab !== entry.tab) {
15884
+ journal.entries.push(entry);
15885
+ }
15886
+ }
15887
+ journal.entries = journal.entries.slice(-RECOVERY_JOURNAL_MAX_ENTRIES);
15888
+ while (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES && journal.entries.length > 0) {
15889
+ journal.entries.shift();
15890
+ }
15891
+ if (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES) return { ok: false, reason: "capacity" };
15892
+ const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
15893
+ try {
15894
+ writeFileSync(temp, `${JSON.stringify(journal)}\n`, { mode: 0o600, flag: "wx" });
15895
+ renameSync(temp, path);
15896
+ return { ok: true, changed: true };
15897
+ } catch (error) {
15898
+ try { unlinkSync(temp); } catch { /* only our temporary file */ }
15899
+ return { ok: false, reason: error?.code ?? "unwritable" };
15900
+ }
15901
+ });
15902
+ }
15903
+
15904
+ function safeRecordRecoveryTransition(scope, event, write = recordRecoveryTransition) {
15905
+ try { return write(scope, event); } catch { return { ok: false, reason: "unavailable" }; }
15906
+ }
15907
+
15908
+ function recoveryLedgerSummary(scope, resource, nowMs = Date.now()) {
15909
+ if (!scope?.path || !RATE_RESOURCES.includes(resource)) return null;
15910
+ const loaded = readGovernorState(scope.path, nowMs, { persistMigration: false });
15911
+ if (!loaded.ok) return null;
15912
+ const state = loaded.value;
15913
+ const debt = state.debt[resource];
15914
+ const debtUnits = debt.unresolvedUnits + debt.quiescentUnits;
15915
+ const charged = governorChargedCost(state, resource, nowMs);
15916
+ const oldest = Object.values(state.debtGroups)
15917
+ .filter((group) => !group.quiescent && group.units[resource] > 0)
15918
+ .map((group) => group.at);
15919
+ return {
15920
+ receiptUnits: Number.isSafeInteger(charged) ? charged - debtUnits : null,
15921
+ debtUnits,
15922
+ oldestUnresolvedAt: oldest.length > 0 ? Math.min(...oldest) : null,
15923
+ observerAt: state.observers[resource]?.at > 0 && state.observers[resource].at <= nowMs
15924
+ ? state.observers[resource].at : null,
15925
+ debtCount: debt.unresolvedCount + debt.quiescentCount,
15926
+ receiptCount: Object.values(state.reservations).filter((reservation) =>
15927
+ reservationCost(reservation, resource, state.leases, nowMs) > 0).length +
15928
+ (reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs) > 0 ? 1 : 0),
15929
+ };
15930
+ }
15931
+
14459
15932
  function acquisitionFailureCopy(reason) {
14460
15933
  if (reason === "busy") return { short: "lock busy", narrow: "Shared lock busy", full: "Acquisition store is busy; checking the shared lock" };
14461
15934
  if (reason === "unwritable") return { short: "store blocked", narrow: "Store blocked", full: "Acquisition store cannot be written; check config permissions" };
@@ -14557,25 +16030,6 @@ function measuredNestedOperationCost(operation, value) {
14557
16030
  return actual;
14558
16031
  }
14559
16032
 
14560
- function uncertainReservationReceipts(scope, reservationId, nowMs) {
14561
- const snapshot = inspectGovernor(scope, nowMs);
14562
- const reservation = snapshot.ok ? snapshot.value.reservations?.[reservationId] : null;
14563
- const accessKey = scope.accessKey ?? scope.hash;
14564
- if (!reservation || typeof accessKey !== "string") return [];
14565
- return RATE_RESOURCES.flatMap((resource) => {
14566
- if (reservation.costs[resource] <= 0) return [];
14567
- return [{
14568
- id: `${reservationId}:${resource}`,
14569
- reservationId,
14570
- accessKey,
14571
- resource,
14572
- epoch: reservation.epochs[resource],
14573
- units: reservation.costs[resource],
14574
- startedAt: reservation.startedAt,
14575
- }];
14576
- });
14577
- }
14578
-
14579
16033
  function abortableDelay(ms, signal, {
14580
16034
  setTimeout: setTimeout_ = setTimeout,
14581
16035
  clearTimeout: clearTimeout_ = clearTimeout,
@@ -14770,30 +16224,38 @@ async function runAdmittedOperation({
14770
16224
  }
14771
16225
  const reservationId = admitted.value.reservationId;
14772
16226
  const settlementScope = { ...scope, identityProvider: null };
14773
- const pendingReceipts = uncertainReservationReceipts(scope, reservationId, now());
16227
+ const transportScope = { ...scope, receiptCapability: admitted.value.receiptCapability };
14774
16228
  try {
14775
- const value = await requestIdentityStorage.run(scope, () => run(signal));
16229
+ const value = await requestIdentityStorage.run(transportScope, () => run(signal));
14776
16230
  const measured = measuredNestedOperationCost(operation, value);
14777
16231
  const costs = measured ?? operationCost(operation);
14778
- const outcome = value?.ok === false ? governorOutcomeForError(value.failure) : "measured-success";
14779
- completeReservation(settlementScope, reservationId, {
16232
+ const outcome = value?.notStarted === true ? "measured-success" :
16233
+ value?.ok === false ? governorOutcomeForError(value.failure) : "measured-success";
16234
+ const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
14780
16235
  outcome,
14781
- ...(measured || outcome === "measured-success" ? { actualCost: costs } : {}),
14782
- }, now());
16236
+ ...(value?.notStarted === true ? { neverStarted: true } : {}),
16237
+ ...(measured || outcome === "measured-success" ? { actualCosts: costs } : {}),
16238
+ }, now);
16239
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
16240
+ return { ok: false, error: new Error(`API receipt settlement ${terminal.status}`),
16241
+ reservationId };
16242
+ }
14783
16243
  const identityCoordinator = scope.identityCoordinator ?? runtimeIdentityCoordinator;
14784
16244
  if (scope.accessKey && scope.accessKey !== identityCoordinator?.current()?.accessKey) {
14785
16245
  return { ok: false, error: new Error("Credential changed"), reservationId };
14786
16246
  }
14787
- return { ok: true, value, reservationId,
14788
- uncertainReceipts: measured ? [] : pendingReceipts };
16247
+ return { ok: true, value, reservationId };
14789
16248
  } catch (error) {
14790
16249
  const actualCost = measuredNestedOperationCost(operation, { requestMetrics: error?.requestMetrics });
14791
- const uncertainReceipts = actualCost ? [] : pendingReceipts;
14792
- completeReservation(settlementScope, reservationId, {
16250
+ const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
14793
16251
  outcome: governorOutcomeForError(error),
14794
- ...(actualCost ? { actualCost } : {}),
14795
- }, now());
14796
- return { ok: false, error, reservationId, uncertainReceipts };
16252
+ ...(error?.notStarted === true ? { neverStarted: true } : {}),
16253
+ ...(actualCost ? { actualCosts: actualCost } : {}),
16254
+ }, now);
16255
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
16256
+ error.receiptSettlement = terminal.status;
16257
+ }
16258
+ return { ok: false, error, reservationId };
14797
16259
  }
14798
16260
  }
14799
16261
 
@@ -15010,6 +16472,7 @@ function refreshStatus({
15010
16472
  stale = false,
15011
16473
  disconnected = false,
15012
16474
  screenReader = false,
16475
+ onDemand = false,
15013
16476
  } = {}) {
15014
16477
  const status = (kind, glyphKind, label, tone, animate = false, detailKind = null) => ({
15015
16478
  kind,
@@ -15046,6 +16509,7 @@ function refreshStatus({
15046
16509
  // Keep the cause visible until a successful result clears it.
15047
16510
  if (activeError) return status("failed", "failed", "Failed", "attention");
15048
16511
  if (securityIncomplete) return status("limited", "limited", "Limited", "attention");
16512
+ if (onDemand) return status("cached", "watching", "Cached", "inert");
15049
16513
  if (sharedData) return status("shared", "watching", "Shared", "inert", false, "sharing");
15050
16514
  if (["waiting", "pending", "probe"].includes(mode)) {
15051
16515
  const sharing = sharedLaneProvenance(governorDecision).waitCause === "shared-lane";
@@ -15535,6 +16999,8 @@ function App({ onCreateRemote = () => {} } = {}) {
15535
16999
  const [waiting, setWaiting] = useState({ actions: true, issues: true, prs: true, security: true });
15536
17000
  const [governorDecisions, setGovernorDecisions] = useState(probingGovernorDecisions);
15537
17001
  const [acquisitionFailures, setAcquisitionFailures] = useState({});
17002
+ const [recoveryCauses, setRecoveryCauses] = useState({ core: null, graphql: null });
17003
+ const recoveryRef = useRef(recoveryCauses);
15538
17004
  const [governorEpochs, setGovernorEpochs] = useState(null);
15539
17005
  const [requestStatuses, setRequestStatuses] = useState(() =>
15540
17006
  Object.fromEntries(TABS.map((candidate) => [candidate.key, null])),
@@ -15560,6 +17026,10 @@ function App({ onCreateRemote = () => {} } = {}) {
15560
17026
  ? { mode: "paused", reason: activeFailure, coordinationError: true,
15561
17027
  cause: acquisitionFailureCopy(activeFailure).short }
15562
17028
  : governorDecisions[tab.key];
17029
+ const activeResource = RATE_RESOURCES.find((resource) => tabRequestCost(tab.key)[resource] > 0) ?? null;
17030
+ const retainedRecovery = activeResource ? recoveryCauses[activeResource] : null;
17031
+ const activeRecovery = retainedRecovery?.origin === "source" && retainedRecovery.tab !== tab.key
17032
+ ? null : retainedRecovery;
15563
17033
  const activeRequestStatus = requestStatuses[tab.key];
15564
17034
  // Once any endpoint proves the folder has no remote, the whole dashboard is
15565
17035
  // in setup mode. Keeping this tab-local made a quick tab switch replace the
@@ -15590,7 +17060,10 @@ function App({ onCreateRemote = () => {} } = {}) {
15590
17060
  // Counts the lines actually rendered, not the notes collected -- getting this
15591
17061
  // wrong by one row is what makes ink repaint the whole frame.
15592
17062
  const extraLines =
15593
- NOTICE_ROWS +
17063
+ (activeRecovery || tabError || activeGovernorDecision?.coordinationError ||
17064
+ activeGovernorDecision?.mode === "paused" || activeGovernorDecision?.disconnected ||
17065
+ data[tab.key] !== null && !runtimeIdentityCoordinator?.current()
17066
+ ? 3 : NOTICE_ROWS) +
15594
17067
  (tab.key === "security" && !remoteSetup ? securityLines.length : 0);
15595
17068
  // Reserve lines for: the tab bar and the divider under it (2), the panel's
15596
17069
  // top and bottom edges (2), the column header and its separator (2), and
@@ -15641,6 +17114,14 @@ function App({ onCreateRemote = () => {} } = {}) {
15641
17114
  const ageMs = collectorCachedSourceAge(cachedTab);
15642
17115
  return ageMs === null ? [] : [[candidate.key, { ageMs, measuredAt: performance.now() }]];
15643
17116
  })));
17117
+ // Keep server source timestamps separate from client-local age estimates.
17118
+ // An SSH receipt's age is measured on the client and cannot prove a new
17119
+ // source observation when a cached snapshot is delivered again.
17120
+ const collectorSourceSuccessRef = useRef(Object.fromEntries(TABS.flatMap((candidate) => {
17121
+ const cachedTab = cachedEntry?.tabs[candidate.key];
17122
+ const sourceAt = cachedTab?.source?.lastSuccessAt ?? cachedTab?.lastOk;
17123
+ return Number.isFinite(sourceAt) ? [[candidate.key, sourceAt]] : [];
17124
+ })));
15644
17125
  const fetchTabRef = useRef(null);
15645
17126
  const contextCoordinatorRef = useRef(null);
15646
17127
  const governorRef = useRef(null);
@@ -16106,6 +17587,53 @@ function App({ onCreateRemote = () => {} } = {}) {
16106
17587
  : { ...current, [key]: value });
16107
17588
  }
16108
17589
 
17590
+ const scheduledControlAttempts = new Map();
17591
+
17592
+ function recordRuntimeRecovery(key, reason, { retryAt = null, origin = "coordination" } = {}) {
17593
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
17594
+ if (!resource) return;
17595
+ const at = Date.now();
17596
+ const previous = recoveryRef.current[resource];
17597
+ const candidate = recoveryCause({
17598
+ reason, resource, at, retryAt, sourceAt: lastOkRef.current[key] ?? null,
17599
+ origin, tab: origin === "source" ? key : null,
17600
+ });
17601
+ const hardFailure = ["corrupt", "unwritable", "capacity", "accounting-overflow"].includes(candidate.code);
17602
+ if (!hardFailure && previous?.code === "receipt-retry" &&
17603
+ terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
17604
+ if (!hardFailure && previous?.code === "request-queue" &&
17605
+ cleanupQueueForResource(cleanupQueue, resource) > 0) return;
17606
+ if (previous?.origin === "source" && previous.tab === key && origin === "coordination" &&
17607
+ !["corrupt", "unwritable", "capacity", "accounting-overflow",
17608
+ "receipt-retry", "interrupted-request"].includes(candidate.code)) return;
17609
+ if (previous?.code === candidate.code && previous?.retryAt === candidate.retryAt &&
17610
+ previous?.origin === candidate.origin && previous?.tab === candidate.tab &&
17611
+ at - previous.at < 60_000) return;
17612
+ const cause = recoveryCause({ ...candidate, reason: candidate.code,
17613
+ terminalizationBacklog: cleanupQueueForResource(cleanupQueue, resource) +
17614
+ terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource),
17615
+ ...recoveryLedgerSummary(scope, resource, at) });
17616
+ cause.firstAt = previous?.code === cause.code && previous?.origin === cause.origin &&
17617
+ previous?.tab === cause.tab ? previous.firstAt : at;
17618
+ recoveryRef.current = { ...recoveryRef.current, [resource]: cause };
17619
+ setRecoveryCauses(recoveryRef.current);
17620
+ if (scope) setTimeout(() => { safeRecordRecoveryTransition(scope, cause); }, 0).unref?.();
17621
+ }
17622
+
17623
+ function clearRuntimeRecovery(resource, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
17624
+ const previous = recoveryRef.current[resource];
17625
+ if (previous?.code === "receipt-retry" &&
17626
+ terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
17627
+ if (previous?.code === "request-queue" &&
17628
+ cleanupQueueForResource(cleanupQueue, resource) > 0) return;
17629
+ if (!shouldClearRecoveryCause(previous, { sourceSuccess, tab, scheduledHold })) return;
17630
+ recoveryRef.current = { ...recoveryRef.current, [resource]: null };
17631
+ setRecoveryCauses(recoveryRef.current);
17632
+ if (scope) setTimeout(() => {
17633
+ safeRecordRecoveryTransition(scope, { resource, clear: true, at: Date.now() });
17634
+ }, 0).unref?.();
17635
+ }
17636
+
16109
17637
  function publishGovernorEpochs(snapshot) {
16110
17638
  const epochs = snapshot?.value?.epochs;
16111
17639
  if (!isRecord(epochs)) return;
@@ -16135,7 +17663,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16135
17663
  return null;
16136
17664
  }
16137
17665
 
16138
- function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false) {
17666
+ function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false, retryAt = null) {
17667
+ recordRuntimeRecovery(key, acquisitionFailure && reason === "busy" ? "acquisition-busy" : reason,
17668
+ { retryAt });
16139
17669
  if (acquisitionFailure) setAcquisitionFailures((current) => current[key] === reason
16140
17670
  ? current : { ...current, [key]: reason ?? "unavailable" });
16141
17671
  const hold = diagnosticHold ?? (
@@ -16180,16 +17710,27 @@ function App({ onCreateRemote = () => {} } = {}) {
16180
17710
  return;
16181
17711
  }
16182
17712
  if (!snapshot?.ok) {
16183
- pauseCoordination(key, snapshot?.reason);
17713
+ pauseCoordination(key, snapshot?.reason, null, false, snapshot?.retryAt);
16184
17714
  return;
16185
17715
  }
16186
17716
  const tabReady = governorTabControlReady(snapshot, key, nowMs);
16187
17717
  if (!refreshed?.ok && !tabReady) {
16188
- pauseCoordination(key, refreshed?.reason, "observer");
17718
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0 &&
17719
+ snapshot.value.observers[candidate]?.outcome !== "healthy");
17720
+ pauseCoordination(key, refreshed?.reason, "observer", false,
17721
+ resource ? snapshot.value.observers[resource]?.nextAt : null);
16189
17722
  return;
16190
17723
  }
16191
17724
  publishGovernorEpochs(snapshot);
16192
17725
  if (refreshed.value?.status === "waiting" && !tabReady) {
17726
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
17727
+ if (resource && !recoveryRef.current[resource] && scope) {
17728
+ const shared = readRecoveryTransitions(scope).current[resource];
17729
+ if (shared?.origin === "coordination") {
17730
+ recoveryRef.current = { ...recoveryRef.current, [resource]: shared };
17731
+ setRecoveryCauses(recoveryRef.current);
17732
+ }
17733
+ }
16193
17734
  setTabGovernorDecision(key, { mode: "waiting", probing: true });
16194
17735
  return;
16195
17736
  }
@@ -16198,32 +17739,45 @@ function App({ onCreateRemote = () => {} } = {}) {
16198
17739
  const reservation = pendingIntent
16199
17740
  ? snapshot.value.reservations[`reservation:${pendingIntent.intentId}`]
16200
17741
  : null;
17742
+ const costs = tabRequestCost(key);
16201
17743
  if (reservation?.status === "scheduled" && reservation.notBefore > nowMs) {
17744
+ const accessKey = pendingIntent.cleanupScope?.accessKey;
17745
+ const reservationId = `reservation:${pendingIntent.intentId}`;
17746
+ const lastAttempt = scheduledControlAttempts.get(key);
17747
+ if (accessKey && pendingIntent.cleanupScope?.hash === scope?.hash &&
17748
+ identity()?.accessKey === accessKey &&
17749
+ (lastAttempt?.id !== reservationId || nowMs - lastAttempt.at >= 30_000)) {
17750
+ scheduledControlAttempts.set(key, { id: reservationId, at: nowMs });
17751
+ // A guarded hold write can retire an older transient notice. Merely
17752
+ // reading a future grant or an absent hold cannot prove recovery.
17753
+ const cleared = clearScheduledRuntimeControlHold(acquisitionEngine,
17754
+ acquisitionSubscriptions, key, accessKey, nowMs);
17755
+ for (const resource of RATE_RESOURCES) {
17756
+ if (costs[resource] > 0) clearRuntimeRecovery(resource, { scheduledHold: cleared });
17757
+ }
17758
+ }
16202
17759
  setTabGovernorDecision(key, visibleGovernorDecision({
16203
17760
  status: "scheduled",
16204
- reservationId: `reservation:${pendingIntent.intentId}`,
17761
+ reservationId,
16205
17762
  ...reservation,
16206
17763
  ...currentSharedLaneProvenance(pendingIntent, snapshot.value.leases, nowMs),
16207
17764
  }));
16208
17765
  return;
16209
17766
  }
16210
- const costs = tabRequestCost(key);
16211
17767
  for (const resource of RATE_RESOURCES) {
16212
17768
  if (costs[resource] <= 0) continue;
16213
17769
  const budget = snapshot.value.budgets[resource];
16214
17770
  const decision = budget
16215
- ? resourceDecision({
16216
- budget,
16217
- resource,
16218
- leases: snapshot.value.leases,
16219
- reservations: Object.values(snapshot.value.reservations),
16220
- nowMs,
16221
- cost: costs[resource],
16222
- })
17771
+ ? governorResourceDecision(snapshot.value, resource, nowMs, costs[resource])
16223
17772
  : { mode: "probe", reason: "budget-unknown" };
16224
17773
  if (decision.mode === "open") continue;
16225
17774
  const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision.reason) ||
16226
17775
  budget?.blockUntil > nowMs;
17776
+ const holdReason = budget?.blockUntil > nowMs
17777
+ ? budget.blockReason ?? decision.reason : decision.reason;
17778
+ recordRuntimeRecovery(key, holdReason, {
17779
+ retryAt: recoveryRetryAt(decision, budget, snapshot.value.observers[resource], nowMs),
17780
+ });
16227
17781
  setRuntimeAcquisitionHold(
16228
17782
  acquisitionEngine,
16229
17783
  acquisitionSubscriptions,
@@ -16235,7 +17789,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16235
17789
  );
16236
17790
  setTabGovernorDecision(key, {
16237
17791
  mode: resetHold || decision.mode === "paused" ? "paused" : "waiting",
16238
- reason: decision.reason,
17792
+ reason: holdReason,
16239
17793
  resetMs: budget?.resetMs ?? decision.resetMs ?? null,
16240
17794
  probing: decision.mode === "probe" && !resetHold,
16241
17795
  });
@@ -16249,6 +17803,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16249
17803
  identity()?.accessKey ?? null,
16250
17804
  );
16251
17805
  setTabGovernorDecision(key, null);
17806
+ for (const resource of RATE_RESOURCES) {
17807
+ if (costs[resource] > 0) clearRuntimeRecovery(resource);
17808
+ }
16252
17809
  }
16253
17810
 
16254
17811
  async function refreshRuntimeBudget(currentScope) {
@@ -16260,15 +17817,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16260
17817
  return refreshed;
16261
17818
  }
16262
17819
 
16263
- function reconcileRuntimeUncertainty(currentScope, snapshot) {
17820
+ function syncRuntimeQuotaProjection(currentScope, snapshot) {
16264
17821
  if (!snapshot?.ok) return;
16265
- const evidence = Object.fromEntries(RATE_RESOURCES.flatMap((resource) => {
16266
- const observer = snapshot.value.observers?.[resource];
16267
- const epoch = snapshot.value.epochs?.[resource];
16268
- return observer?.outcome === "healthy" && typeof epoch === "string" && Number.isFinite(observer.at)
16269
- ? [[resource, { epoch, observedAt: observer.at }]] : [];
16270
- }));
16271
- acquisitionEngine.reconcileUncertainty(currentScope.accessKey, evidence);
17822
+ acquisitionEngine.syncQuotaProjection(currentScope);
16272
17823
  }
16273
17824
 
16274
17825
  function commit(key, run, {
@@ -16320,7 +17871,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16320
17871
  : run;
16321
17872
  return requestIdentityStorage.run(scope, fencedRun)
16322
17873
  .then(async (result) => {
16323
- settleReservationWithBudgetObservations(settlementScope, leaseId, reservationId, result?.measuredSuccess === false
17874
+ const terminal = queueTerminalization(settlementScope, scope.receiptCapability, result?.measuredSuccess === false
16324
17875
  ? { outcome: "rejected", observations: result?.observations ?? [] }
16325
17876
  : {
16326
17877
  outcome: "measured-success",
@@ -16329,7 +17880,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16329
17880
  graphql: result?.graphqlSpent ?? GRAPHQL_PER_FETCH[key] ?? 0,
16330
17881
  },
16331
17882
  observations: result?.observations ?? [],
16332
- }, Date.now());
17883
+ });
17884
+ if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
17885
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
17886
+ failAcquisition(acquisition, { requestMetrics: acquisitionRequestMetrics(result), hold: "primary" });
17887
+ pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
17888
+ return;
17889
+ }
16333
17890
  if (!currentAccess()) {
16334
17891
  failAcquisition(acquisition, {
16335
17892
  requestMetrics: acquisitionRequestMetrics(result),
@@ -16352,7 +17909,6 @@ function App({ onCreateRemote = () => {} } = {}) {
16352
17909
  acquisition.capabilities = result.capabilities ?? {};
16353
17910
  acquisition.repositoryIdentity = result.repositoryIdentity;
16354
17911
  acquisition.requestMetrics = acquisitionRequestMetrics(result);
16355
- acquisition.uncertainReceipts = result.uncertainReceipts ?? [];
16356
17912
  }
16357
17913
  // Identical payload: skip the parse *and* the state update. Returning
16358
17914
  // the same state object makes React bail out of the re-render, so an
@@ -16387,9 +17943,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16387
17943
  // so the next admitted tick parses its response instead of taking the
16388
17944
  // identical-output fast path.
16389
17945
  if (transition.kind === "unusable") {
17946
+ recordRuntimeRecovery(key, "source-incomplete", { origin: "source" });
16390
17947
  failAcquisition(acquisition, {
16391
17948
  requestMetrics: acquisition?.requestMetrics ?? {},
16392
- uncertainReceipts: acquisition?.uncertainReceipts ?? [],
16393
17949
  hold: "primary",
16394
17950
  });
16395
17951
  rawRef.current[key] = transition.nextRaw;
@@ -16402,9 +17958,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16402
17958
  // freshness. Do not retain its raw value either, so the next source
16403
17959
  // retry is parsed instead of taking the identical-payload fast path.
16404
17960
  if (transition.kind === "blind") {
17961
+ recordRuntimeRecovery(key, "security-incomplete", { origin: "source" });
16405
17962
  failAcquisition(acquisition, {
16406
17963
  requestMetrics: acquisition?.requestMetrics ?? {},
16407
- uncertainReceipts: acquisition?.uncertainReceipts ?? [],
16408
17964
  hold: "primary",
16409
17965
  });
16410
17966
  rawRef.current[key] = transition.nextRaw;
@@ -16439,6 +17995,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16439
17995
  publishStagedEntities(entityRef.current, stagedEntities, transition.kind);
16440
17996
  if (transition.kind === "unchanged") {
16441
17997
  lastOkRef.current[key] = completedAt;
17998
+ for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
17999
+ clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
18000
+ }
16442
18001
  clearBackoff(`tab:${key}`);
16443
18002
  setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
16444
18003
  const cachedTab = pick(
@@ -16455,6 +18014,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16455
18014
  const tabData = transition.data;
16456
18015
  const tabMeta = transition.meta;
16457
18016
  lastOkRef.current[key] = completedAt;
18017
+ for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
18018
+ clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
18019
+ }
16458
18020
  clearBackoff(`tab:${key}`);
16459
18021
  rawRef.current[key] = transition.nextRaw;
16460
18022
  setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
@@ -16474,10 +18036,14 @@ function App({ onCreateRemote = () => {} } = {}) {
16474
18036
  requestMetrics: err?.requestMetrics ?? {},
16475
18037
  hold: acquisitionFailureHold(err),
16476
18038
  });
16477
- settleReservationWithBudgetObservations(settlementScope, leaseId, reservationId, {
18039
+ const terminal = queueTerminalization(settlementScope, scope.receiptCapability, {
16478
18040
  outcome: governorOutcomeForError(err),
16479
18041
  observations: err?.budgetObservations ?? [],
16480
- }, Date.now());
18042
+ });
18043
+ if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
18044
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
18045
+ pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
18046
+ }
16481
18047
  if (cancelled || !currentAccess() || err?.name === "AbortError") return;
16482
18048
  // Preserve both the verdict and the bounded raw error in state. The
16483
18049
  // renderer translates recognized verdicts at draw time, which lets a
@@ -16494,6 +18060,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16494
18060
  // recover on the very next tick once the network is back.
16495
18061
  const steps = verdict === "rate-limited" ? null : pick(FAILURE_LADDER, verdict, null);
16496
18062
  if (steps) recordFailure(`tab:${key}`, performance.now(), steps);
18063
+ const backoffUntil = alertBackoff.get(backoffStorageKey(`tab:${key}`))?.until;
18064
+ const retryAt = Number.isFinite(backoffUntil)
18065
+ ? Date.now() + Math.max(0, backoffUntil - performance.now()) : null;
18066
+ if (terminal.status !== "blocked" && terminal.status !== "compacted") {
18067
+ const recovery = requestFailureRecovery(err, verdict);
18068
+ recordRuntimeRecovery(key, recovery.reason, { origin: recovery.origin, retryAt });
18069
+ }
16497
18070
  if (verdict === "rate-limited") {
16498
18071
  const snapshot = inspectGovernor(scope, Date.now());
16499
18072
  const costs = tabRequestCost(key);
@@ -16508,6 +18081,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16508
18081
  });
16509
18082
  }
16510
18083
  handleRateLimitBlocks(key, blocks, Date.now());
18084
+ const held = inspectGovernor(scope, Date.now());
18085
+ const heldUntil = RATE_RESOURCES.filter((resource) => costs[resource] > 0)
18086
+ .map((resource) => held.value?.budgets?.[resource]?.blockUntil)
18087
+ .filter((deadline) => Number.isFinite(deadline) && deadline > Date.now());
18088
+ if (heldUntil.length > 0) recordRuntimeRecovery(key, "rate-limited", {
18089
+ origin: "source", retryAt: Math.min(...heldUntil),
18090
+ });
16511
18091
  }
16512
18092
  })
16513
18093
  .finally(() => {
@@ -16541,6 +18121,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16541
18121
  const leaseId = governorId();
16542
18122
  const pending = new Map();
16543
18123
  const cleanupQueue = new Set();
18124
+ let lastWallAt = Date.now();
16544
18125
  // One manual handoff per tab, carrying which key produced it. A keypress
16545
18126
  // during an automatic request must not be lost, but repeated keypresses
16546
18127
  // during the manual request itself must not create a trailing second batch.
@@ -16579,12 +18160,20 @@ function App({ onCreateRemote = () => {} } = {}) {
16579
18160
  const unchangedPolls = Object.fromEntries(TAB_KEYS.map((key) => [key, 0]));
16580
18161
  let pendingBlockPublications = new Map();
16581
18162
  const acquisitionSubscriptions = new Map();
18163
+ // One-shot demand survives admission delays, but ends when an observation arrives.
18164
+ const requestedTabs = new Map();
16582
18165
 
16583
18166
  function adoptAcquisitionSnapshot(key, snapshot, receipt = null) {
16584
18167
  if (!snapshot || cancelled) return;
18168
+ if (requestedTabs.has(key) && !queuedManual.has(key) &&
18169
+ snapshot.lastSuccessAt >= requestedTabs.get(key)) {
18170
+ requestedTabs.delete(key);
18171
+ if (runtime.background === "on-demand" && key !== "actions") pollDueAt[key] = Infinity;
18172
+ }
16585
18173
  if (runtime.connect === null && Number.isFinite(snapshot.nextDueAt)) {
16586
18174
  sharedSnapshotDueAt = { ...sharedSnapshotDueAt, [key]: snapshot.nextDueAt };
16587
- if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key) {
18175
+ if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key &&
18176
+ Number.isFinite(pollIntervalFor(key, key))) {
16588
18177
  const at = Date.now();
16589
18178
  const due = sharedAcquisitionPollDeadline(pollDueAt[key], snapshot.nextDueAt, at, true);
16590
18179
  if (due < pollDueAt[key]) {
@@ -16657,6 +18246,14 @@ function App({ onCreateRemote = () => {} } = {}) {
16657
18246
  const host = effectiveRuntimeHost({ remoteUrls: runtimeRemoteUrls });
16658
18247
  const handles = new Map();
16659
18248
  const receivedSnapshots = new Set();
18249
+ const initialKey = TABS[activeIndexRef.current].key;
18250
+ const collectorDemand = (key) => ({
18251
+ active: TABS[activeIndexRef.current].key === key &&
18252
+ (runtime.background !== "on-demand" || key === "actions"),
18253
+ background: runtime.background === "all",
18254
+ floorMs: runtime.refreshMs,
18255
+ pages: pageStateRef.current[key]?.pages ?? 1,
18256
+ });
16660
18257
  let collectorReady = false;
16661
18258
  setGovernorDecisions(Object.fromEntries(TAB_KEYS.map((key) => [key, { disconnected: true }])));
16662
18259
  if (!repository || !host) {
@@ -16686,39 +18283,46 @@ function App({ onCreateRemote = () => {} } = {}) {
16686
18283
  },
16687
18284
  });
16688
18285
  for (const key of TAB_KEYS) {
16689
- const demand = {
16690
- active: TABS[activeIndexRef.current].key === key,
16691
- background: runtime.background !== "off",
16692
- floorMs: runtime.refreshMs,
16693
- pages: pageStateRef.current[key]?.pages ?? 1,
16694
- };
18286
+ const demand = collectorDemand(key);
16695
18287
  const subscribed = client.subscribe({ host, repo: repository, resource: key, demand,
16696
18288
  sourceCheckpoint: cachedEntry?.tabs[key]?.source ?? null,
16697
18289
  onSnapshot: (snapshot, receipt) => {
16698
18290
  receivedSnapshots.add(key);
18291
+ const sourceAdvanced = collectorSourceAdvanced(
18292
+ collectorSourceSuccessRef.current[key], snapshot.lastSuccessAt);
16699
18293
  adoptAcquisitionSnapshot(key, snapshot, receipt);
18294
+ if (Number.isFinite(snapshot.lastSuccessAt) &&
18295
+ (!Number.isFinite(collectorSourceSuccessRef.current[key]) ||
18296
+ snapshot.lastSuccessAt > collectorSourceSuccessRef.current[key])) {
18297
+ collectorSourceSuccessRef.current[key] = snapshot.lastSuccessAt;
18298
+ }
18299
+ if (sourceAdvanced) {
18300
+ for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
18301
+ clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
18302
+ }
18303
+ }
16700
18304
  setErrors((current) => current[key] === null ? current : { ...current, [key]: null });
16701
18305
  setWaiting((current) => current[key] ? { ...current, [key]: false } : current);
16702
18306
  setGovernorDecisions((current) => ({ ...current, [key]:
16703
18307
  collectorDisplayDecision({ connected: collectorReady, hasSnapshot: true }) }));
16704
18308
  },
16705
- onHold: (hold) => setGovernorDecisions((current) => ({ ...current, [key]:
16706
- collectorDisplayDecision({ connected: collectorReady,
16707
- hasSnapshot: receivedSnapshots.has(key), hold }) })),
18309
+ onHold: (hold) => {
18310
+ if (hold === "app-auth") recordRuntimeRecovery(key, "app-auth", { origin: "source" });
18311
+ setGovernorDecisions((current) => ({ ...current, [key]:
18312
+ collectorDisplayDecision({ connected: collectorReady,
18313
+ hasSnapshot: receivedSnapshots.has(key), hold }) }));
18314
+ },
16708
18315
  });
16709
18316
  if (subscribed.ok) handles.set(key, subscribed.value);
16710
18317
  }
18318
+ if (runtime.background === "on-demand" && initialKey !== "actions") {
18319
+ handles.get(initialKey)?.refresh();
18320
+ }
16711
18321
  fetchTabRef.current = (key, request = {}) => {
16712
18322
  const handle = handles.get(key);
16713
18323
  if (!handle) return;
16714
- if (request.kind === "tab-switch") {
16715
- for (const [candidate, candidateHandle] of handles) {
16716
- candidateHandle.updateDemand({ active: candidate === key, background: runtime.background !== "off",
16717
- floorMs: runtime.refreshMs, pages: pageStateRef.current[candidate]?.pages ?? 1 });
16718
- }
16719
- } else {
16720
- handle.updateDemand({ active: true, background: runtime.background !== "off",
16721
- floorMs: runtime.refreshMs, pages: pageStateRef.current[key]?.pages ?? 1 });
18324
+ for (const [candidate, candidateHandle] of handles) {
18325
+ candidateHandle.updateDemand(collectorDemand(candidate));
16722
18326
  }
16723
18327
  handle.refresh(request.force === true);
16724
18328
  };
@@ -16795,12 +18399,11 @@ function App({ onCreateRemote = () => {} } = {}) {
16795
18399
  demand: acquisition.demand?.active ? "active" : "inactive",
16796
18400
  unchangedCount: publicationView.unchangedCount,
16797
18401
  inProgressCI: key === "actions" && actionsInProgress(rows),
16798
- background: runtime.background,
18402
+ background: runtime.background === "on-demand" ? "all" : runtime.background,
16799
18403
  }),
16800
18404
  hold: null,
16801
18405
  capabilities: acquisition.capabilities ?? {},
16802
18406
  requestMetrics: acquisition.requestMetrics ?? {},
16803
- uncertainReceipts: acquisition.uncertainReceipts ?? [],
16804
18407
  repositoryIdentity: acquisition.repositoryIdentity,
16805
18408
  meta,
16806
18409
  securityNotes: key === "security"
@@ -16841,10 +18444,17 @@ function App({ onCreateRemote = () => {} } = {}) {
16841
18444
  automaticStatusVisible) {
16842
18445
  const descriptor = tabForKey(key);
16843
18446
  let acquisitionStartAttempted = false;
18447
+ let transportScope = null;
16844
18448
  const transported = await runStartedAcquisitionTransport(
16845
18449
  () => {
16846
18450
  const admitted = inspectGovernor(currentScope, nowMs);
16847
18451
  if (!admitted.ok) return admitted;
18452
+ const admittedReservation = admitted.value.reservations[reservationId];
18453
+ if (!admittedReservation?.receipt || admittedReservation.status !== "started") {
18454
+ return { ok: false, reason: "compacted" };
18455
+ }
18456
+ transportScope = { ...currentScope,
18457
+ receiptCapability: receiptCapabilityFromReservation(reservationId, admittedReservation) };
16848
18458
  acquisitionStartAttempted = true;
16849
18459
  const receipt = {
16850
18460
  reservationId,
@@ -16875,13 +18485,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16875
18485
  force: item.force,
16876
18486
  catalog: workflowCatalogRef.current,
16877
18487
  pages: item.acquisition.demand?.pages ?? pageStateRef.current[key]?.pages ?? 1,
16878
- governor: { scope: currentScope, leaseId },
18488
+ governor: { scope: transportScope, leaseId },
16879
18489
  previousRaw: rawRef.current[key] ?? null,
16880
18490
  }),
16881
18491
  {
16882
18492
  force: item.force,
16883
18493
  manual: item.manual === true,
16884
- scope: currentScope,
18494
+ scope: transportScope,
16885
18495
  reservationId,
16886
18496
  admittedAt: nowMs,
16887
18497
  automaticStatusVisible,
@@ -16892,6 +18502,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16892
18502
  },
16893
18503
  );
16894
18504
  if (transported.ok) return transported;
18505
+ if (transportScope?.receiptCapability) {
18506
+ const terminal = queueTerminalization(currentScope,
18507
+ transportScope.receiptCapability, { outcome: "rejected", neverStarted: true });
18508
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
18509
+ pauseCoordination(key, terminal.reason ?? terminal.status, null, true);
18510
+ }
18511
+ }
16895
18512
  pauseCoordination(key, transported.reason, null, acquisitionStartAttempted);
16896
18513
  // The governor reservation is already started. Even a transient failed
16897
18514
  // claim write cannot safely rebind it; retain its charge and get a fresh
@@ -16915,6 +18532,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16915
18532
  demand: key === activeKey ? "active" : "inactive",
16916
18533
  unchangedCount: unchangedPolls[key],
16917
18534
  inProgressCI: key === "actions" && actionsInProgress(actionRows),
18535
+ requested: requestedTabs.has(key),
16918
18536
  background: runtime.background,
16919
18537
  });
16920
18538
  }
@@ -16925,7 +18543,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16925
18543
  const localDueAt = Number.isFinite(step) ? at + step : Number.POSITIVE_INFINITY;
16926
18544
  pollDueAt = {
16927
18545
  ...pollDueAt,
16928
- [key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active),
18546
+ [key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active && Number.isFinite(step)),
16929
18547
  };
16930
18548
  }
16931
18549
 
@@ -16937,6 +18555,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16937
18555
  function openPollDeadlines(at) {
16938
18556
  const activeKey = TABS[activeIndexRef.current].key;
16939
18557
  const opened = { ...NEVER_DUE };
18558
+ if (runtime.background === "on-demand") requestedTabs.set(activeKey, at);
16940
18559
  opened[activeKey] = at + 1;
16941
18560
  let slot = 1;
16942
18561
  for (const key of TAB_KEYS) {
@@ -17038,6 +18657,8 @@ function App({ onCreateRemote = () => {} } = {}) {
17038
18657
  coordinator.invalidate();
17039
18658
  setFailureContext(null);
17040
18659
  if (discardData) {
18660
+ recoveryRef.current = { core: null, graphql: null };
18661
+ setRecoveryCauses(recoveryRef.current);
17041
18662
  rawRef.current = {};
17042
18663
  entityRef.current.clear();
17043
18664
  lastOkRef.current = {};
@@ -17115,6 +18736,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17115
18736
  }
17116
18737
  scope = createQuotaScope(current, { root: runtimeIdentityCoordinator.root, identityProvider: identity });
17117
18738
  cleanupScope = { ...scope, identityProvider: null };
18739
+ const savedRecovery = readRecoveryTransitions(scope);
18740
+ recoveryRef.current = savedRecovery.status === "healthy"
18741
+ ? { ...savedRecovery.current } : { core: null, graphql: null };
18742
+ setRecoveryCauses(recoveryRef.current);
17118
18743
  const nextTarget = dashboardCacheTarget({ repo: runtime.repo, ghRepo: process.env.GH_REPO, host: current.host, account: current.accessKey });
17119
18744
  // The mount-time target had to guess the account, and guesses
17120
18745
  // "unverified" whenever the bounded warm resolution above did not land
@@ -17176,6 +18801,11 @@ function App({ onCreateRemote = () => {} } = {}) {
17176
18801
  function finishPendingCleanup(key, item) {
17177
18802
  if (pending.get(key) === item) pending.delete(key);
17178
18803
  cleanupQueue.delete(item);
18804
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
18805
+ if (resource && cleanupQueueForResource(cleanupQueue, resource) === 0 &&
18806
+ recoveryRef.current[resource]?.code === "request-queue") {
18807
+ clearRuntimeRecovery(resource);
18808
+ }
17179
18809
  setAcquisitionFailures((current) => {
17180
18810
  if (!Object.hasOwn(current, key)) return current;
17181
18811
  const next = { ...current };
@@ -17204,7 +18834,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17204
18834
  }
17205
18835
 
17206
18836
  function retryPendingCleanup(key, item, at = Date.now()) {
17207
- if (item.cleanupInFlight) return false;
18837
+ if (item.cleanupInFlight) {
18838
+ recordRuntimeRecovery(key, "request-queue");
18839
+ return false;
18840
+ }
17208
18841
  item.cleanupPending = true;
17209
18842
  item.cleanupKey = key;
17210
18843
  cleanupQueue.add(item);
@@ -17230,6 +18863,7 @@ function App({ onCreateRemote = () => {} } = {}) {
17230
18863
  finishPendingCleanup(key, item);
17231
18864
  }
17232
18865
  if (!item.governorClean) armWake("data", governorControlRetryAt(at, runtime.refreshMs), dataWake);
18866
+ if (pending.has(key)) recordRuntimeRecovery(key, "request-queue");
17233
18867
  return !pending.has(key);
17234
18868
  }
17235
18869
 
@@ -17252,7 +18886,7 @@ function App({ onCreateRemote = () => {} } = {}) {
17252
18886
  // the tab's failure ladder. `force` is the separate, stronger request that
17253
18887
  // also drops validators -- the `R` key. `r` is manual and not forced, which
17254
18888
  // is what makes a quiet refresh cost nothing.
17255
- async function requestTab(key, kind = "active", { force = false } = {}) {
18889
+ async function requestTab(key, kind = "active", { force = false, detachTransport = false } = {}) {
17256
18890
  const manual = kind === "manual";
17257
18891
  const signal = controller.signal;
17258
18892
  const monotonicNow = performance.now();
@@ -17333,7 +18967,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17333
18967
  currentScope.accessKey,
17334
18968
  );
17335
18969
  if (ownership.value.role !== "producer") {
17336
- if (ownership.value.snapshot) adoptAcquisitionSnapshot(key, ownership.value.snapshot);
18970
+ if (ownership.value.snapshot) {
18971
+ if (ownership.value.reason === "fresh") requestedTabs.delete(key);
18972
+ adoptAcquisitionSnapshot(key, ownership.value.snapshot);
18973
+ }
17337
18974
  if (ownership.value.sharingCount > 1) {
17338
18975
  setTabGovernorDecision(key, {
17339
18976
  mode: "waiting",
@@ -17457,9 +19094,19 @@ function App({ onCreateRemote = () => {} } = {}) {
17457
19094
  setTabWaiting(key, false);
17458
19095
  const item = pending.get(key);
17459
19096
  if (!item) return { persisted: false, retry: false, kind, key };
17460
- await startPendingAcquisitionTransport(
19097
+ const transport = startPendingAcquisitionTransport(
17461
19098
  key, item, currentScope, decision.reservationId, nowMs, signal, false,
17462
19099
  );
19100
+ if (detachTransport) {
19101
+ // The scheduling single-flight owns admission, not the HTTP lifetime.
19102
+ // A background response must not block the next active-tab wake.
19103
+ void transport.catch((error) => {
19104
+ if (cancelled || pending.get(key) !== item) return;
19105
+ item.retryOnCleanup = true;
19106
+ cancelPending(key, currentScope);
19107
+ pauseCoordination(key, error?.reason ?? "stale");
19108
+ });
19109
+ } else await transport;
17463
19110
  return { persisted: true, retry: false, kind, key };
17464
19111
  }
17465
19112
  fetchTabRef.current = (key, { force = false, kind = force ? "manual" : "active" } = {}) => {
@@ -17470,6 +19117,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17470
19117
  unchangedPolls[key] = advanceUnchangedCount(unchangedPolls[key], "subscribed");
17471
19118
  }
17472
19119
  const requestedAt = Date.now();
19120
+ if (runtime.background === "on-demand") {
19121
+ if (kind === "tab-switch") requestedTabs.clear();
19122
+ requestedTabs.set(key, requestedAt);
19123
+ }
17473
19124
  // Manual work replaces the active poll that would otherwise be due. Move
17474
19125
  // that deadline before touching the governor so a data wake already in
17475
19126
  // this event-loop turn cannot start a second batch as soon as the forced
@@ -17583,11 +19234,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17583
19234
  const budget = snapshot.value.budgets[resource];
17584
19235
  const observer = snapshot.value.observers[resource];
17585
19236
  const decision = budget
17586
- ? availableForGrant({ budget, resource, nowMs })
19237
+ ? governorAvailableForGrant(snapshot.value, resource, nowMs)
17587
19238
  : { mode: "paused" };
17588
- const held = pendingBlockPublications.has(resource) ||
17589
- budget?.blockUntil > nowMs ||
17590
- decision.mode === "paused" && decision.reason === "budget-exhausted";
19239
+ const held = governorResourceHeld(snapshot.value, resource, nowMs,
19240
+ pendingBlockPublications.has(resource));
17591
19241
  const retryAt = budget?.blockUntil > nowMs
17592
19242
  ? budget.blockUntil
17593
19243
  : observer?.nextAt ?? decision.retryAt ?? wakes.controlAt;
@@ -17607,13 +19257,14 @@ function App({ onCreateRemote = () => {} } = {}) {
17607
19257
  states: Object.fromEntries(TAB_KEYS.map((key) => [key, {
17608
19258
  unchangedCount: unchangedPolls[key],
17609
19259
  inProgressCI: key === "actions" && actionsInProgress(),
19260
+ requested: requestedTabs.has(key),
17610
19261
  }])),
17611
19262
  });
17612
19263
  scheduledWakePolls = planned.due;
17613
19264
  pollDueAt = planned.dueAt;
17614
19265
  backgroundIndex = planned.backgroundIndex;
17615
19266
  const outcomes = await Promise.allSettled(
17616
- planned.due.map(({ key, kind }) => requestTab(key, kind)),
19267
+ planned.due.map(({ key, kind }) => requestTab(key, kind, { detachTransport: true })),
17617
19268
  );
17618
19269
  const retryAt = governorControlRetryAt(Date.now(), runtime.refreshMs);
17619
19270
  const retried = retryFailedPollBatch({ due: planned.due, outcomes,
@@ -17666,6 +19317,13 @@ function App({ onCreateRemote = () => {} } = {}) {
17666
19317
 
17667
19318
  async function runControlWake() {
17668
19319
  if (cancelled) return;
19320
+ const controlAt = Date.now();
19321
+ const clockCause = wallClockRecovery(lastWallAt, controlAt, Math.min(runtime.refreshMs, 5_000));
19322
+ lastWallAt = controlAt;
19323
+ if (clockCause) {
19324
+ recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
19325
+ setNow(new Date(controlAt));
19326
+ }
17669
19327
  for (const item of [...cleanupQueue]) retryPendingCleanup(item.cleanupKey, item);
17670
19328
  await runtimeIdentityCoordinator.refresh();
17671
19329
  if (cancelled) return;
@@ -17684,11 +19342,11 @@ function App({ onCreateRemote = () => {} } = {}) {
17684
19342
  }
17685
19343
  const refreshed = currentScope
17686
19344
  ? await refreshRuntimeBudget(currentScope)
17687
- : { ok: false, reason: runtimeIdentityCoordinator?.inspect()?.reason ?? "stale" };
19345
+ : { ok: false, ...(runtimeIdentityCoordinator?.inspect() ?? { reason: "stale" }) };
17688
19346
  if (cancelled) return;
17689
19347
  const checkedAt = Date.now();
17690
19348
  const snapshot = currentScope ? inspectGovernor(currentScope, checkedAt) : refreshed;
17691
- if (currentScope) reconcileRuntimeUncertainty(currentScope, snapshot);
19349
+ if (currentScope) syncRuntimeQuotaProjection(currentScope, snapshot);
17692
19350
  if (currentScope && snapshot.ok && pendingBlockPublications.size > 0) {
17693
19351
  attemptPendingBlockPublications(currentScope, checkedAt, snapshot.value);
17694
19352
  }
@@ -17750,14 +19408,27 @@ function App({ onCreateRemote = () => {} } = {}) {
17750
19408
  const livenessWake = createRecurringWake({
17751
19409
  scheduler: wakeScheduler,
17752
19410
  intervalMs: Math.min(runtime.refreshMs, 5000),
17753
- run: async () => {
19411
+ run: () => {
17754
19412
  if (cancelled || remoteSetupRef.current) return;
17755
19413
  const at = Date.now();
19414
+ const clockCause = wallClockRecovery(lastWallAt, at, Math.min(runtime.refreshMs, 5_000));
19415
+ lastWallAt = at;
19416
+ for (const resource of RATE_RESOURCES) {
19417
+ if (["receipt-retry", "request-queue"].includes(recoveryRef.current[resource]?.code)) {
19418
+ clearRuntimeRecovery(resource);
19419
+ }
19420
+ }
19421
+ if (clockCause) {
19422
+ recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
19423
+ setNow(new Date(at));
19424
+ void controlWake();
19425
+ return;
19426
+ }
17756
19427
  if (wakeScheduler.at("control") <= at ||
17757
- !Number.isFinite(wakeScheduler.at("control"))) await controlWake();
19428
+ !Number.isFinite(wakeScheduler.at("control"))) void controlWake();
17758
19429
  if (pending.size > 0 || cleanupQueue.size > 0 ||
17759
19430
  wakeScheduler.at("data") <= Date.now() ||
17760
- Object.values(pollDueAt).some((due) => due <= Date.now())) await dataWake();
19431
+ Object.values(pollDueAt).some((due) => due <= Date.now())) void dataWake();
17761
19432
  },
17762
19433
  onError: (error) => {
17763
19434
  if (!cancelled) pauseCoordination(TABS[activeIndexRef.current].key, error?.reason ?? "stale");
@@ -17773,7 +19444,9 @@ function App({ onCreateRemote = () => {} } = {}) {
17773
19444
  if (cancelled) return;
17774
19445
  const currentScope = ensureScope(Date.now());
17775
19446
  if (!currentScope) {
17776
- pauseCoordination(TABS[activeIndexRef.current].key, runtimeIdentityCoordinator?.inspect()?.reason ?? "unknown-scope");
19447
+ const identityFailure = runtimeIdentityCoordinator?.inspect();
19448
+ pauseCoordination(TABS[activeIndexRef.current].key,
19449
+ identityFailure?.reason ?? "unknown-scope", null, false, identityFailure?.retryAt);
17777
19450
  armWake("control", governorControlRetryAt(Date.now(), runtime.refreshMs), controlWake);
17778
19451
  return;
17779
19452
  }
@@ -17783,7 +19456,7 @@ function App({ onCreateRemote = () => {} } = {}) {
17783
19456
  if (cancelled) return;
17784
19457
  const checkedAt = Date.now();
17785
19458
  const snapshot = inspectGovernor(currentScope, checkedAt);
17786
- reconcileRuntimeUncertainty(currentScope, snapshot);
19459
+ syncRuntimeQuotaProjection(currentScope, snapshot);
17787
19460
  publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
17788
19461
  if (snapshot.ok) {
17789
19462
  controlEpochs = { ...snapshot.value.epochs };
@@ -17907,7 +19580,14 @@ function App({ onCreateRemote = () => {} } = {}) {
17907
19580
  const displayError = formatTabErrorForWidth(tabError, failureContext, Math.max(1, cols - 5));
17908
19581
  const coordinationError = activeGovernorDecision?.coordinationError && !remoteSetup;
17909
19582
  const coordinationReason = activeFailure ?? activeGovernorDecision?.reason ?? "unavailable";
17910
- const coordinationCondition = coordinationError ? `${tab.key}\0${coordinationReason}` : null;
19583
+ const noticeReason = activeRecovery?.code ??
19584
+ (tabError?.kind === "fetch" ? tabError.verdict : null) ??
19585
+ (coordinationError || activeGovernorDecision?.mode === "paused" ? coordinationReason : null) ??
19586
+ (statusDisconnected ? runtime.connect === null ? "identity-unavailable"
19587
+ : activeGovernorDecision?.reason === "app-auth" ? "app-auth" : "disconnected" : null);
19588
+ // A continuous fault stays sustained when its explanation becomes more
19589
+ // specific. Only losing the fault (or switching tabs) restarts the debounce.
19590
+ const coordinationCondition = !remoteSetup && noticeReason ? tab.key : null;
17911
19591
  useEffect(() => {
17912
19592
  setVisibleCoordinationCondition(null);
17913
19593
  if (coordinationCondition === null) return;
@@ -17919,10 +19599,21 @@ function App({ onCreateRemote = () => {} } = {}) {
17919
19599
  }, [coordinationCondition]);
17920
19600
  const showCoordinationNotice = coordinationCondition !== null &&
17921
19601
  visibleCoordinationCondition === coordinationCondition;
17922
- const noticeLine = showCoordinationNotice
17923
- ? activeFailure ? acquisitionFailureCopy(activeFailure)[cols < 40 ? "narrow" : "full"]
17924
- : coordinationNotice(coordinationReason)
17925
- : !remoteSetup && displayError ? displayError : "";
19602
+ const noticeResource = activeResource;
19603
+ const noticeSourceAt = collectorSourceAgeRef.current[tab.key]
19604
+ ? now.getTime() - collectorSourceAgeRef.current[tab.key].ageMs -
19605
+ Math.max(0, performance.now() - collectorSourceAgeRef.current[tab.key].measuredAt)
19606
+ : lastOkRef.current[tab.key] ?? null;
19607
+ const noticeCause = activeRecovery ?? (noticeReason ? recoveryCause({
19608
+ reason: noticeReason === "other" ? "network-outage" : noticeReason,
19609
+ resource: noticeResource, sourceAt: noticeSourceAt,
19610
+ retryAt: activeGovernorDecision?.retryAt ?? activeGovernorDecision?.resetMs ??
19611
+ activeGovernorDecision?.notBefore ?? null,
19612
+ origin: tabError ? "source" : "coordination", tab: tabError ? tab.key : null,
19613
+ }) : null);
19614
+ const noticeLines = showCoordinationNotice && noticeCause
19615
+ ? presentRecovery(noticeCause, { nowMs: now.getTime(), cols })
19616
+ : [!remoteSetup && displayError ? displayError : ""];
17926
19617
  const noticeTone = showCoordinationNotice ? ATTENTION : displayError ? ERROR_TEXT : undefined;
17927
19618
  const spin = SPINNER[frame % SPINNER.length];
17928
19619
 
@@ -17999,6 +19690,22 @@ function App({ onCreateRemote = () => {} } = {}) {
17999
19690
  });
18000
19691
  }
18001
19692
 
19693
+ if (runtime.background === "on-demand" && tab.key !== "actions" && data[tab.key] !== null &&
19694
+ ["watching", "shared", "stale"].includes(semanticStatus.kind) && !tabError &&
19695
+ !activeFailure && !(tab.key === "security" && securityBlind)) {
19696
+ semanticStatus = refreshStatus({ onDemand: true });
19697
+ }
19698
+
19699
+ // A settled on-demand view has no data wake to advance its visible age.
19700
+ // Tick only its display clock; source timestamps and acquisition stay idle.
19701
+ const showCachedAge = semanticStatus.kind === "cached" && staleFor !== null;
19702
+ useEffect(() => {
19703
+ if (!showCachedAge) return;
19704
+ setNow(new Date());
19705
+ const timer = setInterval(() => setNow(new Date()), 1000);
19706
+ return () => clearInterval(timer);
19707
+ }, [showCachedAge]);
19708
+
18002
19709
  const allItems = items ?? [];
18003
19710
  const tabOffsetRaw = offset[tab.key] ?? 0;
18004
19711
  const selectedKey = selected[tab.key] ?? null;
@@ -18199,7 +19906,8 @@ function App({ onCreateRemote = () => {} } = {}) {
18199
19906
  !showHelp &&
18200
19907
  tooNarrow &&
18201
19908
  e(Text, { dimColor: true, wrap: "truncate-end" }, "too narrow"),
18202
- e(Text, { color: noticeTone, wrap: "truncate-end" }, noticeLine),
19909
+ ...noticeLines.map((line, index) =>
19910
+ e(Text, { key: `recovery:${index}`, color: noticeTone, wrap: "truncate-end" }, line)),
18203
19911
  !showHelp &&
18204
19912
  !tooNarrow &&
18205
19913
  !remoteSetup &&
@@ -18294,7 +20002,8 @@ function App({ onCreateRemote = () => {} } = {}) {
18294
20002
  status: semanticStatus,
18295
20003
  detail: activeGovernorDecision,
18296
20004
  spin: showSpinner ? spin : null,
18297
- stale: staleLabel,
20005
+ stale: semanticStatus.kind === "cached" && staleFor != null
20006
+ ? formatDuration(Math.max(0, staleFor)) : staleLabel,
18298
20007
  nowMs: now.getTime(),
18299
20008
  interactive,
18300
20009
  cols: frameCols,
@@ -18589,6 +20298,7 @@ export {
18589
20298
  resolveEffectiveCredential,
18590
20299
  identityRegistryRoot,
18591
20300
  inspectIdentityRegistry,
20301
+ doctorCachedQuotaScope,
18592
20302
  claimIdentityBootstrap,
18593
20303
  finishIdentityBootstrap,
18594
20304
  createIdentityCoordinator,
@@ -18633,6 +20343,10 @@ export {
18633
20343
  budgetEpoch,
18634
20344
  resourceReserve,
18635
20345
  availableForGrant,
20346
+ governorChargedCost,
20347
+ governorAvailableForGrant,
20348
+ governorResourceDecision,
20349
+ governorResourceHeld,
18636
20350
  nextExternalFactor,
18637
20351
  resourceDecision,
18638
20352
  governorPhaseOffset,
@@ -18658,6 +20372,7 @@ export {
18658
20372
  serializeGovernorState,
18659
20373
  readGovernorState,
18660
20374
  writeGovernorState,
20375
+ writeGovernorQuotaState,
18661
20376
  pidIsDead,
18662
20377
  claimGovernorLock,
18663
20378
  releaseGovernorLock,
@@ -18685,6 +20400,9 @@ export {
18685
20400
  readIntentDecision,
18686
20401
  cancelIntent,
18687
20402
  startReservation,
20403
+ issueGovernorDispatch,
20404
+ terminalGovernorDispatch,
20405
+ acknowledgeSealedGeneration,
18688
20406
  completeReservation,
18689
20407
  settleReservationWithBudgetObservations,
18690
20408
  recordResourceBlock,
@@ -18846,7 +20564,9 @@ export {
18846
20564
  acquisitionRequestMetrics,
18847
20565
  acquisitionFailureHold,
18848
20566
  setRuntimeAcquisitionHold,
20567
+ clearScheduledRuntimeControlHold,
18849
20568
  loadAcquisitionStore,
20569
+ readAcquisitionSnapshot,
18850
20570
  createAcquisitionEngine,
18851
20571
  pollPolicyInterval,
18852
20572
  advanceUnchangedCount,
@@ -18882,11 +20602,27 @@ export {
18882
20602
  runtimeIntentGate,
18883
20603
  rateLimitBlockDecision,
18884
20604
  coordinationNotice,
20605
+ recoveryCause,
20606
+ shouldClearRecoveryCause,
20607
+ recoveryRetryAt,
20608
+ terminalizationRecoveryCode,
20609
+ terminalizationBacklogForResource,
20610
+ receiptCapabilityFromReservation,
20611
+ requestFailureRecoveryCode,
20612
+ requestFailureRecovery,
20613
+ wallClockRecovery,
20614
+ collectorSourceAdvanced,
20615
+ cleanupQueueForResource,
20616
+ presentRecovery,
20617
+ recordRecoveryTransition,
20618
+ safeRecordRecoveryTransition,
20619
+ readRecoveryTransitions,
18885
20620
  mergeRateLimitBlockPublications,
18886
20621
  hydrateRateLimitBlockPublication,
18887
20622
  retryRateLimitBlockPublication,
18888
20623
  rateLimitBlockProbeRecovered,
18889
20624
  admitGovernorOperation,
20625
+ queueTerminalization,
18890
20626
  runAdmittedOperation,
18891
20627
  abortableDelay,
18892
20628
  awaitCollectorReservation,
@@ -18912,6 +20648,7 @@ export {
18912
20648
  demandedPageCount,
18913
20649
  mergeDemandedPages,
18914
20650
  operationPausedUntil,
20651
+ pauseOperation,
18915
20652
  LIST_LIMIT,
18916
20653
  readGraphqlObserver,
18917
20654
  GRAPHQL_QUERIES,