gh-glance 0.15.2 → 0.16.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/CHANGELOG.md +69 -1
  2. package/README.md +76 -25
  3. package/index.mjs +2257 -501
  4. package/package.json +1 -1
package/index.mjs CHANGED
@@ -22,7 +22,7 @@
22
22
  // disarmDevBuildLeak() for how that path is kept survivable rather than fatal.
23
23
  process.env.NODE_ENV ??= "production";
24
24
 
25
- import { execFile, spawn } from "node:child_process";
25
+ import { execFile, execFileSync, spawn } from "node:child_process";
26
26
  import { AsyncLocalStorage } from "node:async_hooks";
27
27
  import { createHash, createHmac, createPrivateKey, randomUUID, sign as cryptoSign, timingSafeEqual } from "node:crypto";
28
28
  import { EventEmitter } from "node:events";
@@ -207,7 +207,7 @@ const STALE_AFTER_MS = 30_000;
207
207
  // A coordination hold must persist long enough to be meaningful before it
208
208
  // takes the notice row. Short startup and lock-contention states recover on
209
209
  // their own and should not present themselves as user-visible failures.
210
- const COORDINATION_NOTICE_AFTER_MS = 2_000;
210
+ const COORDINATION_NOTICE_AFTER_MS = 800;
211
211
 
212
212
  // Mutable because the fetchers below are defined before argv is parsed, and the
213
213
  // argv block is what fills this in. Everything here has a working default, so
@@ -221,7 +221,7 @@ const runtime = {
221
221
  host: null,
222
222
  repoExplicit: false,
223
223
  refreshMs: REFRESH_MS,
224
- background: "all",
224
+ background: "on-demand",
225
225
  verbose: false,
226
226
  connect: null,
227
227
  initialTabIndex: 0,
@@ -734,7 +734,9 @@ const pausedOperations = new Map();
734
734
 
735
735
  function pauseOperation(operation, untilMs) {
736
736
  if (!Number.isFinite(untilMs)) return;
737
- pausedOperations.set(operation, Math.max(pausedOperations.get(operation) ?? 0, untilMs));
737
+ const until = Math.max(pausedOperations.get(operation) ?? 0, untilMs);
738
+ pausedOperations.set(operation, until);
739
+ return () => { if (pausedOperations.get(operation) === until) pausedOperations.delete(operation); };
738
740
  }
739
741
 
740
742
  function operationPausedUntil(operation, now = Date.now()) {
@@ -744,7 +746,8 @@ function operationPausedUntil(operation, now = Date.now()) {
744
746
  return until;
745
747
  }
746
748
 
747
- async function runGh(args, { signal, operation, input = null, execute = null } = {}) {
749
+ async function runGh(args, { signal, operation, input = null, execute = null,
750
+ closeWaitMs = 15_000 } = {}) {
748
751
  if (operationCost(operation) === null) {
749
752
  throw new Error(`undeclared gh operation: ${operation ?? "missing"}`);
750
753
  }
@@ -756,11 +759,32 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
756
759
  const requestNow = typeof bound?.now === "function" ? bound.now : Date.now;
757
760
  const permit = identityCoordinator && !local
758
761
  ? await acquireIdentityHttpPermit(identityCoordinator, { signal, now: requestNow,
762
+ operationDeadline: CONTROL_OPERATIONS.includes(operation) ? undefined : bound?.receiptCapability?.deadline,
759
763
  ...(typeof bound?.httpWait === "function" ? { wait: bound.httpWait } : {}) }) : null;
760
764
  const startedAt = requestNow();
761
765
  let requestError = null;
762
766
  let requestStdout = null;
763
767
  let requestStarted = false;
768
+ let knownNeverStarted = false;
769
+ let requestSettled = false;
770
+ let successfulPromise = false;
771
+ let dispatchEvidence = null;
772
+ let dispatchCapability = null;
773
+ let childCloseProof = null;
774
+ let activeDispatchKey = null;
775
+ let activeDispatchReleased = false;
776
+ const releaseActiveDispatch = () => {
777
+ if (!activeDispatchKey || activeDispatchReleased) return;
778
+ activeDispatchReleased = true;
779
+ const remaining = (activeOwnerDispatches.get(activeDispatchKey) ?? 1) - 1;
780
+ if (remaining > 0) activeOwnerDispatches.set(activeDispatchKey, remaining);
781
+ else activeOwnerDispatches.delete(activeDispatchKey);
782
+ if (dispatchCapability && !dispatchCapability.controlResource) {
783
+ const sealed = inspectGovernor({ ...bound, identityProvider: null }, requestNow());
784
+ const nonce = sealed.ok && sealed.value.debtGroups[dispatchCapability.generation]?.nonce;
785
+ if (nonce) acknowledgeSealedGeneration(bound, dispatchCapability, nonce, requestNow());
786
+ }
787
+ };
764
788
  try {
765
789
  assertBoundCredential(bound);
766
790
  if (identityCoordinator && operation === "budget-core-observer") {
@@ -789,6 +813,23 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
789
813
  }
790
814
  }
791
815
  assertBoundCredential(bound);
816
+ if (!local && RATE_RESOURCES.some((resource) => operationCost(operation)[resource] > 0)) {
817
+ dispatchCapability = operation === "budget-core-observer"
818
+ ? control?.value?.receiptCapability
819
+ : operation === "graphql-observer"
820
+ ? bound?.controlReceiptCapability : bound?.receiptCapability;
821
+ if (dispatchCapability) {
822
+ activeDispatchKey = ownerDispatchKey(bound, dispatchCapability);
823
+ activeOwnerDispatches.set(activeDispatchKey,
824
+ (activeOwnerDispatches.get(activeDispatchKey) ?? 0) + 1);
825
+ }
826
+ const issued = issueGovernorDispatch(bound, dispatchCapability, operation, requestNow());
827
+ if (!issued.ok) {
828
+ releaseActiveDispatch();
829
+ throw new Error(`API receipt paused (${issued.reason})`);
830
+ }
831
+ dispatchEvidence = issued.value;
832
+ }
792
833
  const executeChild = execute ?? bound?.executeGh ?? execFileAsync;
793
834
  const pending = executeChild("gh", args, {
794
835
  timeout: GH_TIMEOUT_MS,
@@ -798,6 +839,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
798
839
  signal,
799
840
  });
800
841
  requestStarted = true;
842
+ if (Number.isSafeInteger(pending.child?.pid) &&
843
+ typeof pending.child.once === "function") {
844
+ childCloseProof = new Promise((resolve) => pending.child.once("close", resolve));
845
+ }
846
+ if (dispatchEvidence && Number.isSafeInteger(pending.child?.pid)) {
847
+ markGovernorDispatchChild(bound, dispatchCapability,
848
+ dispatchEvidence.sequence, pending.child.pid, requestNow());
849
+ }
801
850
  if (typeof input === "string") {
802
851
  // A child killed by the timeout or the abort signal closes stdin under
803
852
  // us; that EPIPE is the kill's consequence, not the failure worth
@@ -805,16 +854,73 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
805
854
  pending.child.stdin?.on("error", () => {});
806
855
  pending.child.stdin?.end(input);
807
856
  }
808
- const { stdout } = await pending;
857
+ let stdout;
858
+ try {
859
+ ({ stdout } = await pending);
860
+ requestSettled = true;
861
+ successfulPromise = true;
862
+ } catch (error) {
863
+ requestSettled = true;
864
+ throw error;
865
+ }
809
866
  requestStdout = stdout;
810
- logGh(args, startedAt, `ok ${stdout.length}B`);
811
- return stdout;
812
867
  } catch (err) {
813
868
  requestError = err;
869
+ if (!requestStarted && !dispatchEvidence) err.notStarted ??= true;
870
+ knownNeverStarted = !requestStarted && err?.notStarted === true;
814
871
  if (requestStarted && isRecord(err)) err.httpStarted = true;
815
872
  logGh(args, startedAt, `FAILED ${shortErr(err)}`);
816
873
  throw err;
817
874
  } finally {
875
+ let childClosed = !requestStarted;
876
+ if (requestSettled && childCloseProof) {
877
+ let timer;
878
+ childClosed = await Promise.race([
879
+ childCloseProof.then(() => true),
880
+ new Promise((resolve) => { timer = setTimeout(() => resolve(false), closeWaitMs); }),
881
+ ]);
882
+ clearTimeout(timer);
883
+ }
884
+ if (requestSettled && !childClosed) {
885
+ requestError ??= Object.assign(new Error("GitHub child close unverified"), { httpStarted: true });
886
+ }
887
+ if (successfulPromise) {
888
+ logGh(args, startedAt, childClosed
889
+ ? `ok ${requestStdout.length}B` : "FAILED GitHub child close unverified");
890
+ }
891
+ if (dispatchEvidence && (requestSettled && childClosed || knownNeverStarted)) {
892
+ const finishDispatch = () => {
893
+ const result = terminalGovernorDispatch(bound,
894
+ dispatchCapability, dispatchEvidence.sequence, requestNow(),
895
+ { neverStarted: knownNeverStarted });
896
+ if (result.ok || result.reason === "compacted") releaseActiveDispatch();
897
+ return result;
898
+ };
899
+ const terminal = await retryIdentityCompletion(finishDispatch, {
900
+ timeoutMs: 15_000, shouldRetry: () => true,
901
+ });
902
+ if (!terminal.ok && identityCoordinator?.deferCompletion) {
903
+ identityCoordinator.deferCompletion(
904
+ `dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
905
+ finishDispatch,
906
+ );
907
+ }
908
+ } else if (dispatchEvidence && requestSettled && childCloseProof) {
909
+ // A promise can settle before its subprocess closes. Keep the issued
910
+ // charge until close proves that no more HTTP can escape this child.
911
+ void childCloseProof.then(async () => {
912
+ const finishDispatch = () => terminalGovernorDispatch(bound,
913
+ dispatchCapability, dispatchEvidence.sequence, requestNow());
914
+ const terminal = await retryIdentityCompletion(finishDispatch, {
915
+ timeoutMs: 15_000, shouldRetry: () => true,
916
+ });
917
+ if (terminal.ok || terminal.reason === "compacted") releaseActiveDispatch();
918
+ else identityCoordinator?.deferCompletion?.(
919
+ `dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
920
+ finishDispatch,
921
+ );
922
+ });
923
+ }
818
924
  if (permit) {
819
925
  const release = () => releaseIdentityHttpPermit(identityCoordinator, permit, requestError, requestNow());
820
926
  const released = await retryIdentityCompletion(release, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
@@ -822,11 +928,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
822
928
  }
823
929
  if (control?.ok) {
824
930
  const settle = () => settleIdentityControl(identityCoordinator, control.value,
825
- requestStdout ?? requestError?.stdout, requestNow());
931
+ childClosed ? requestStdout ?? requestError?.stdout : null, requestNow(),
932
+ { neverStarted: !requestStarted && (!dispatchEvidence || knownNeverStarted) });
826
933
  const settled = await retryIdentityCompletion(settle, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
827
934
  if (!settled.ok) identityCoordinator.deferCompletion(`control:${control.value.id}`, settle);
828
935
  }
829
936
  }
937
+ if (successfulPromise && requestError) throw requestError;
938
+ return requestStdout;
830
939
  }
831
940
 
832
941
  function parseGhApiResponse(stdout) {
@@ -1638,7 +1747,6 @@ async function fetchActions(signal, {
1638
1747
  }
1639
1748
  const responses = [runsResponse];
1640
1749
  let requestMetrics = runsResponse.requestMetrics ?? restResponseRequestMetrics(runsResponse);
1641
- const uncertainReceipts = [];
1642
1750
  const runs = parseActionsRuns(runsResponse.body);
1643
1751
  const startedAt = Date.now();
1644
1752
  let nextCatalog = catalog;
@@ -1662,7 +1770,6 @@ async function fetchActions(signal, {
1662
1770
  request,
1663
1771
  }),
1664
1772
  });
1665
- uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
1666
1773
  if (admitted.ok && admitted.value) {
1667
1774
  responses.push(admitted.value);
1668
1775
  requestMetrics = mergeAcquisitionRequestMetrics(
@@ -1695,7 +1802,6 @@ async function fetchActions(signal, {
1695
1802
  failedRequests: requestMetrics.failedRequests ?? 0,
1696
1803
  coreUnitsTotal: requestMetrics.coreUnits ?? 0,
1697
1804
  requestMetrics,
1698
- uncertainReceipts,
1699
1805
  graphqlSpent: GRAPHQL_PER_FETCH.actions,
1700
1806
  stagedEntities: batch.stagedEntities,
1701
1807
  observations: batch.observations,
@@ -1896,7 +2002,6 @@ async function fetchGraphqlList(kind, mapRow, {
1896
2002
  throw error;
1897
2003
  }
1898
2004
  const observations = [...first.observations];
1899
- const uncertainReceipts = [];
1900
2005
  const repositoryIdentity = normalizeRepositoryIdentity(first.data?.repository);
1901
2006
  // Only the first page. Pages past it opened their own reservation inside
1902
2007
  // runAdmittedOperation and settle against it, so adding them here charges the
@@ -1934,7 +2039,6 @@ async function fetchGraphqlList(kind, mapRow, {
1934
2039
  waitMs: GOVERNOR_ADMISSION_WAIT_MS,
1935
2040
  run: (admittedSignal) => fetchPage(kind, { signal: admittedSignal, after: cursor, operation: `page:${kind}` }),
1936
2041
  });
1937
- uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
1938
2042
  // A later page that is denied, fails, or returns errors leaves the rows
1939
2043
  // already gathered exactly as they are. Losing page one because page two
1940
2044
  // was refused would turn a budget decision into data loss.
@@ -1988,7 +2092,6 @@ async function fetchGraphqlList(kind, mapRow, {
1988
2092
  httpRequests: requestMetrics.httpRequests ?? 0,
1989
2093
  failedRequests: requestMetrics.failedRequests ?? 0,
1990
2094
  requestMetrics,
1991
- uncertainReceipts,
1992
2095
  observations,
1993
2096
  incomplete,
1994
2097
  repositoryIdentity,
@@ -2186,6 +2289,12 @@ function sharedLaneEvidence(value) {
2186
2289
  // Below this many completed shared calls in a probe window, external-spend
2187
2290
  // inference is noise. Under the threshold the loop retains the previous factor.
2188
2291
  const MIN_SAMPLE_CALLS = 5;
2292
+ // Pacing divides the spendable share by the external factor, so a large factor
2293
+ // compounds: gh-glance's own paid calls shrink, the next sample's ratio grows,
2294
+ // and the lane is pushed past the reset while quota sits unused. The reserve,
2295
+ // not this divisor, is what keeps gh-glance off the floor, so the divisor can
2296
+ // be bounded without letting external load exhaust the account.
2297
+ const MAX_PACING_EXTERNAL_FACTOR = 4;
2189
2298
 
2190
2299
  const REQUEST_PRIORITIES = Object.freeze({
2191
2300
  manual: 0,
@@ -2265,6 +2374,41 @@ function reservationCost(reservation, resource, leases, nowMs) {
2265
2374
  return Number.isFinite(cost) && cost > 0 ? Math.max(0, cost - accounted) : 0;
2266
2375
  }
2267
2376
 
2377
+ function governorAccountingSafe(state, nowMs) {
2378
+ return RATE_RESOURCES.every((resource) =>
2379
+ Number.isSafeInteger(governorChargedCost(state, resource, nowMs)));
2380
+ }
2381
+
2382
+ function governorChargedCost(state, resource, nowMs) {
2383
+ if (!state || !RATE_RESOURCES.includes(resource)) return Number.NaN;
2384
+ let total = state.debt?.[resource]?.unresolvedUnits +
2385
+ state.debt?.[resource]?.quiescentUnits;
2386
+ if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
2387
+ for (const reservation of [state.controlReceipts?.[resource],
2388
+ ...Object.values(state.reservations ?? {})]) {
2389
+ total += reservationCost(reservation, resource, state.leases, nowMs);
2390
+ if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
2391
+ }
2392
+ return total;
2393
+ }
2394
+
2395
+ function governorAvailableForGrant(state, resource, nowMs) {
2396
+ return availableForGrant({ budget: state?.budgets?.[resource], resource, nowMs,
2397
+ chargedCost: governorChargedCost(state, resource, nowMs) });
2398
+ }
2399
+
2400
+ function governorResourceDecision(state, resource, nowMs, cost) {
2401
+ return resourceDecision({ budget: state?.budgets?.[resource], resource, nowMs, cost,
2402
+ chargedCost: governorChargedCost(state, resource, nowMs) });
2403
+ }
2404
+
2405
+ function governorResourceHeld(state, resource, nowMs, pendingBlock = false) {
2406
+ const budget = state?.budgets?.[resource];
2407
+ const decision = governorAvailableForGrant(state, resource, nowMs);
2408
+ return pendingBlock || budget?.blockUntil > nowMs ||
2409
+ decision.mode === "paused" || decision.mode === "open" && decision.spendable <= 0;
2410
+ }
2411
+
2268
2412
  function availableForGrant({
2269
2413
  budget,
2270
2414
  resource = budget?.resource,
@@ -2379,7 +2523,8 @@ function resourceDecision({
2379
2523
  epoch: capacity.epoch,
2380
2524
  };
2381
2525
  }
2382
- const callsPerMs = capacity.spendable / (capacity.resetMs - nowMs) / externalFactor;
2526
+ const callsPerMs = capacity.spendable / (capacity.resetMs - nowMs) /
2527
+ Math.min(externalFactor, MAX_PACING_EXTERNAL_FACTOR);
2383
2528
  if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) {
2384
2529
  return {
2385
2530
  mode: "paused",
@@ -2517,6 +2662,7 @@ function scheduleIntents({
2517
2662
  leases = {},
2518
2663
  budgets = {},
2519
2664
  reservations = [],
2665
+ debtCosts = { core: 0, graphql: 0 },
2520
2666
  lanes = {},
2521
2667
  cursors = {},
2522
2668
  nowMs,
@@ -2551,7 +2697,7 @@ function scheduleIntents({
2551
2697
 
2552
2698
  const chargedTotals = Object.fromEntries(RATE_RESOURCES.map((resource) => [
2553
2699
  resource,
2554
- reservations.reduce(
2700
+ debtCosts[resource] + reservations.reduce(
2555
2701
  (total, reservation) => total + reservationCost(reservation, resource, leases, nowMs),
2556
2702
  0,
2557
2703
  ),
@@ -2694,17 +2840,17 @@ function scheduleIntents({
2694
2840
  }
2695
2841
 
2696
2842
  // ---------- Credential identities and restart-safe coordination ----------
2697
- const IDENTITY_VERSION = 1;
2843
+ const IDENTITY_VERSION = 2;
2698
2844
  const IDENTITY_ATTEMPT_WINDOW_MS = 15 * 60_000;
2699
2845
  const IDENTITY_ATTEMPT_LIMIT = 3;
2700
2846
  const IDENTITY_HOST_ATTEMPT_LIMIT = 12;
2701
2847
  const IDENTITY_MAX_ATTEMPTS = 4096;
2702
2848
  const HTTP_START_GAP_MS = 250;
2703
2849
  const HTTP_MAX_WAITERS = 128;
2704
- // A permit is reclaimable once it cannot still belong to a live request. The
2705
- // owner-death check alone cannot see a process that is stopped, or one whose
2706
- // release lost its lock, so a permit outliving the subprocess timeout it guards
2707
- // is stale by construction rather than by guess.
2850
+ const HTTP_MAX_PERMITS = 3;
2851
+ // Retain the existing bounded permit lease for managed requests. This bounds
2852
+ // unexpired admissions, not surviving OS children after suspension or owner
2853
+ // death; quota receipts separately retain uncertain transport charges.
2708
2854
  const HTTP_PERMIT_MAX_MS = GH_TIMEOUT_MS + 5_000;
2709
2855
  // The longest a primary window can run. An attempt that never obtained reset
2710
2856
  // evidence is still covered once a full window has demonstrably elapsed since it
@@ -2738,11 +2884,15 @@ function retryableCoordination(reason) {
2738
2884
  return RETRYABLE_COORDINATION_REASONS.includes(reason);
2739
2885
  }
2740
2886
 
2741
- // A dead owner releases its permit immediately; a live owner keeps it only for
2742
- // as long as a request could still be running under it. Without the second
2743
- // test a stopped process, or one whose release never won the lock, holds the
2744
- // single machine-wide permit forever and every pane queues behind it.
2887
+ // Managed permit leases retain their existing age/dead-owner reclamation law.
2888
+ // Reclaiming a slot never retires its quota receipt or proves child quiescence.
2745
2889
  function staleHttpPermit(permit, now, kill = process.kill.bind(process)) {
2890
+ // A dead v1 parent does not prove its untracked child ended. Reserve its
2891
+ // one slot until a subsequent verified boot; two managed slots remain usable.
2892
+ if (permit.kind === "legacy") {
2893
+ const bootId = coordinationBootId();
2894
+ return permit.bootId !== null && bootId !== null && permit.bootId !== bootId;
2895
+ }
2746
2896
  if (pidIsDead(permit.pid, kill)) return true;
2747
2897
  return Number.isFinite(permit.startedAt) && now - permit.startedAt > HTTP_PERMIT_MAX_MS;
2748
2898
  }
@@ -2813,8 +2963,8 @@ function emptyIdentityRegistry() {
2813
2963
  return { version: IDENTITY_VERSION, migration: { activated: false, holdUntil: 0 }, identities: {}, attempts: {}, hosts: {} };
2814
2964
  }
2815
2965
 
2816
- function normalizeIdentityRegistry(raw) {
2817
- if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !== IDENTITY_VERSION ||
2966
+ function normalizeIdentityRegistry(raw, version = IDENTITY_VERSION) {
2967
+ if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !== version ||
2818
2968
  !exactKeys(raw.migration, ["activated", "holdUntil"]) || typeof raw.migration.activated !== "boolean" ||
2819
2969
  !Number.isFinite(raw.migration.holdUntil) || raw.migration.holdUntil < 0 ||
2820
2970
  !isRecord(raw.identities) || !isRecord(raw.attempts) || !isRecord(raw.hosts) ||
@@ -2837,13 +2987,20 @@ function normalizeIdentityRegistry(raw) {
2837
2987
  }
2838
2988
  for (const [host, transport] of Object.entries(raw.hosts)) {
2839
2989
  if (!isRecord(transport)) return null;
2840
- const keys = ["lastStartedAt", "cooldownUntil", "permit"];
2990
+ const keys = ["lastStartedAt", "cooldownUntil", version === 1 ? "permit" : "permits"];
2841
2991
  if (Object.hasOwn(transport, "waiters")) keys.push("waiters");
2842
2992
  if (Object.hasOwn(transport, "throttle")) keys.push("throttle");
2843
2993
  if (!normalizeHost(host) || !exactKeys(transport, keys) ||
2844
2994
  !time(transport.lastStartedAt) || !time(transport.cooldownUntil)) return null;
2845
- if (transport.permit !== null && (!exactKeys(transport.permit, ["pid", "nonce", "startedAt"]) ||
2846
- !Number.isSafeInteger(transport.permit.pid) || transport.permit.pid < 1 || !validGovernorId(transport.permit.nonce) || !time(transport.permit.startedAt))) return null;
2995
+ const permits = version === 1 ? (transport.permit === null ? [] : [transport.permit]) : transport.permits;
2996
+ if (!Array.isArray(permits) || permits.length > HTTP_MAX_PERMITS ||
2997
+ new Set(permits.map((permit) => permit?.nonce)).size !== permits.length ||
2998
+ permits.filter((permit) => permit?.kind === "legacy").length > 1 ||
2999
+ permits.some((permit) => !exactKeys(permit, version === 1
3000
+ ? ["pid", "nonce", "startedAt"] : ["pid", "nonce", "startedAt", "kind", "bootId"]) ||
3001
+ !Number.isSafeInteger(permit.pid) || permit.pid < 1 || !validGovernorId(permit.nonce) || !time(permit.startedAt) ||
3002
+ version !== 1 && (!["request", "bootstrap", "legacy"].includes(permit.kind) ||
3003
+ permit.bootId !== null && (typeof permit.bootId !== "string" || permit.bootId.length > 128)))) return null;
2847
3004
  if (!Object.hasOwn(transport, "throttle")) transport.throttle = emptyTransportThrottle();
2848
3005
  if (!isRecord(transport.throttle) || !exactKeys(transport.throttle, ["attempts", "lastAt", "paused"]) ||
2849
3006
  !Number.isSafeInteger(transport.throttle.attempts) || transport.throttle.attempts < 0 ||
@@ -2874,8 +3031,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
2874
3031
  let state;
2875
3032
  if (raw === null) state = emptyIdentityRegistry();
2876
3033
  else {
2877
- try { state = normalizeIdentityRegistry(JSON.parse(raw)); }
2878
- catch { return { ok: false, reason: "corrupt" }; }
3034
+ try {
3035
+ const parsed = JSON.parse(raw);
3036
+ state = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
3037
+ } catch { return { ok: false, reason: "corrupt" }; }
2879
3038
  }
2880
3039
  if (!state) return { ok: false, reason: "corrupt" };
2881
3040
  // Captured before the prunes below so their deletions still count as a
@@ -2883,6 +3042,19 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
2883
3042
  // funnels through here, and each one used to serialize and rename the
2884
3043
  // registry while holding the one machine-wide lock.
2885
3044
  const before = JSON.stringify(state);
3045
+ if (state.version === 1) {
3046
+ // Keep v1 writable until a live owner commits its response and cooldown.
3047
+ // Returning the shared transient reason preserves cached identity/leases.
3048
+ if (Object.values(state.hosts).some((transport) =>
3049
+ transport.permit && !pidIsDead(transport.permit.pid, kill))) return { ok: false, reason: "busy" };
3050
+ for (const transport of Object.values(state.hosts)) {
3051
+ transport.permits = transport.permit
3052
+ ? [{ ...transport.permit, kind: "legacy", bootId: coordinationBootId() }] : [];
3053
+ delete transport.permit;
3054
+ }
3055
+ state.version = IDENTITY_VERSION;
3056
+ }
3057
+
2886
3058
  for (const transport of Object.values(state.hosts)) {
2887
3059
  transport.waiters = transport.waiters.filter((waiter) => waiter.deadline > now && !pidIsDead(waiter.pid, kill));
2888
3060
  }
@@ -2913,9 +3085,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
2913
3085
  // stops the entry from being retained for the life of the install.
2914
3086
  : budget?.resetMs > attempt.startedAt + IDENTITY_UNCERTAIN_MAX_MS;
2915
3087
  if (!attempt.accounted && !resetCovered) return { ok: true, value: false };
2916
- delete ledger.value.reservations[`reservation:${id}`];
2917
- const write = writeGovernorState(scope.path, ledger.value);
2918
- return write.ok ? { ok: true, value: true } : write;
3088
+ // Retiring registry attempt history never retires quota debt. A reset
3089
+ // proves a counter window changed, not that a suspended child cannot
3090
+ // dispatch after it resumes.
3091
+ return { ok: true, value: true };
2919
3092
  });
2920
3093
  if (!retired.ok) return retired;
2921
3094
  if (retired.value) delete state.attempts[id];
@@ -3010,9 +3183,10 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
3010
3183
  return withIdentityRegistry(root, (state) => {
3011
3184
  const migration = inspectLegacyMigration(root, state, now);
3012
3185
  if (!migration.ok) return migration;
3013
- const transport = state.hosts[host] ??= { lastStartedAt: 0, cooldownUntil: 0, permit: null };
3014
- if (transport.permit && staleHttpPermit(transport.permit, now, kill)) transport.permit = null;
3015
- if (transport.permit || transport.waiters?.length) return { ok: false, reason: "identity-busy", retryAt: now + 1000 };
3186
+ const transport = state.hosts[host] ??= emptyIdentityTransport();
3187
+ transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, now, kill));
3188
+ if (transport.throttle.paused) return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
3189
+ if (transport.permits.some((permit) => permit.kind !== "legacy") || transport.waiters.length) return { ok: false, reason: "identity-busy", retryAt: now + 1000 };
3016
3190
  const attempts = Object.values(state.attempts);
3017
3191
  const recent = attempts.filter((attempt) => attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
3018
3192
  const own = recent.filter((attempt) => attempt.credentialKey === credentialKey && attempt.resource === resource && attempt.exceptional);
@@ -3027,7 +3201,7 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
3027
3201
  const nonce = governorId();
3028
3202
  state.attempts[id] = { credentialKey, host, resource, startedAt: now, retryAt: now + attemptRetryDelayMs(own.length),
3029
3203
  ownerPid: process.pid, nonce, status: "started", quotaKey: null, resetMs: null, accounted: false, imported: false, exceptional: true };
3030
- transport.permit = { pid: process.pid, nonce, startedAt: now };
3204
+ transport.permits.push({ pid: process.pid, nonce, startedAt: now, kind: "bootstrap", bootId: null });
3031
3205
  transport.lastStartedAt = now;
3032
3206
  return { ok: true, value: { id, nonce } };
3033
3207
  }, { now, kill });
@@ -3045,32 +3219,31 @@ function importIdentityDebts(root, state, credentialKey, identity, now) {
3045
3219
  return withGovernorLock(scope, () => {
3046
3220
  const loaded = readIdentityQuotaState(state, scope, now);
3047
3221
  if (!loaded.ok) return loaded;
3222
+ for (const id of Object.keys(loaded.value.importMarkers)) {
3223
+ const attempt = state.attempts[id];
3224
+ if (!attempt || attempt.imported && attempt.quotaKey === identity.quotaKey) {
3225
+ delete loaded.value.importMarkers[id];
3226
+ }
3227
+ }
3048
3228
  const transferred = [];
3049
3229
  for (const [id, attempt] of Object.entries(state.attempts)) {
3050
3230
  if (attempt.credentialKey !== credentialKey || attempt.imported) continue;
3051
- const reservationId = `reservation:${id}`;
3052
- loaded.value.reservations[reservationId] ??= {
3231
+ const legacyControl = {
3053
3232
  leaseId: id, intentId: id, costs: { core: 0, graphql: 0, [attempt.resource]: 1 }, actualCosts: null,
3054
3233
  accountedCosts: { core: 0, graphql: 0, [attempt.resource]: attempt.accounted ? 1 : 0 },
3055
3234
  notBefore: attempt.startedAt, status: "started", epochs: { core: null, graphql: null },
3056
3235
  startedAt: attempt.startedAt, completedAt: null, outcome: null,
3057
3236
  };
3237
+ if (!loaded.value.importMarkers[id] &&
3238
+ !transferReceiptToDebt(loaded.value, legacyControl, now, id)) {
3239
+ return { ok: false, reason: "accounting-overflow" };
3240
+ }
3241
+ if (Object.keys(loaded.value.importMarkers).length >= IDENTITY_MAX_ATTEMPTS &&
3242
+ !loaded.value.importMarkers[id]) return { ok: false, reason: "identity-capacity" };
3243
+ loaded.value.importMarkers[id] = true;
3058
3244
  transferred.push(id);
3059
3245
  }
3060
- // importIdentityDebts writes this ledger before the registry transaction
3061
- // that records the mapping commits. A crash in between leaves a charged
3062
- // receipt here whose attempt is gone from the registry, and nothing else
3063
- // ever looks at it again -- it presents only as a permanently smaller
3064
- // budget. Re-bootstrapping the same credential is exactly when it can be
3065
- // recognised, and a full window since it started proves a reset covered it.
3066
- for (const key of Object.keys(loaded.value.reservations)) {
3067
- const attemptId = /^reservation:(.+)$/.exec(key)?.[1];
3068
- if (!attemptId || state.attempts[attemptId]) continue;
3069
- const reservation = loaded.value.reservations[key];
3070
- if (reservation.startedAt + IDENTITY_UNCERTAIN_MAX_MS > now) continue;
3071
- delete loaded.value.reservations[key];
3072
- }
3073
- const written = writeGovernorState(scope.path, loaded.value);
3246
+ const written = writeGovernorQuotaState(scope.path, loaded.value, now);
3074
3247
  if (!written.ok) return written;
3075
3248
  // Marked transferred only once the receipts are durable. withIdentityRegistry
3076
3249
  // persists the registry whatever this returns, so setting these before the
@@ -3090,7 +3263,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
3090
3263
  const attempt = state.attempts[id];
3091
3264
  if (!attempt || attempt.nonce !== nonce || attempt.credentialKey !== credentialKey) return { ok: false, reason: "stale" };
3092
3265
  const transport = state.hosts[attempt.host];
3093
- if (transport.permit?.nonce === nonce) transport.permit = null;
3266
+ transport.permits = transport.permits.filter((permit) => permit.nonce !== nonce);
3094
3267
  attempt.status = response ? "finished" : "uncertain";
3095
3268
  const parsed = response?.body;
3096
3269
  const evidence = response?.rateLimit;
@@ -3128,7 +3301,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
3128
3301
  ledger.value.epochs.core = observation.budget.epoch;
3129
3302
  ledger.value.observers.core = { etag: typeof response.etag === "string" ? response.etag : null, outcome: "healthy", at: now, nextAt: now + BUDGET_PROBE_MS };
3130
3303
  }
3131
- return writeGovernorState(scope.path, ledger.value);
3304
+ return writeGovernorQuotaState(scope.path, ledger.value, now);
3132
3305
  });
3133
3306
  if (!seeded.ok) return seeded;
3134
3307
  }
@@ -3267,35 +3440,53 @@ function startIdentityControl(coordinator, now = Date.now()) {
3267
3440
  const exceptional = !budget || now >= budget.resetMs + BUDGET_RESET_GRACE_MS;
3268
3441
  if (budget && now >= budget.resetMs && !exceptional) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
3269
3442
  if (!exceptional) {
3270
- const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now), 0);
3443
+ const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now),
3444
+ state.debt.core.unresolvedUnits + state.debt.core.quiescentUnits +
3445
+ reservationCost(state.controlReceipts.core, "core", state.leases, now));
3271
3446
  if (budget.remaining - resourceReserve(budget.limit) - reserved < 1) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
3272
3447
  }
3273
3448
  const recent = Object.values(registry.attempts).filter((attempt) => attempt.credentialKey === identity.credentialKey && attempt.resource === "core" && attempt.exceptional && attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
3274
3449
  const retryAt = Math.max(0, ...recent.map((attempt) => attempt.retryAt), recent.length >= IDENTITY_ATTEMPT_LIMIT ? Math.min(...recent.map((attempt) => attempt.startedAt)) + IDENTITY_ATTEMPT_WINDOW_MS : 0);
3275
3450
  if (exceptional && retryAt > now) return { ok: false, reason: "identity-backoff", retryAt };
3276
- if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS || Object.keys(state.reservations).length >= GOVERNOR_MAX_RESERVATIONS) return { ok: false, reason: "identity-capacity" };
3451
+ if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS) return { ok: false, reason: "identity-capacity" };
3452
+ const priorControl = state.controlReceipts.core;
3453
+ if (priorControl) {
3454
+ if (!retireControlReceiptToDebt(state, "core", now)) return { ok: false, reason: "accounting-overflow" };
3455
+ }
3277
3456
  const id = governorId();
3278
3457
  registry.attempts[id] = { credentialKey: identity.credentialKey, host: identity.host, resource: "core", startedAt: now,
3279
3458
  retryAt: exceptional ? now + attemptRetryDelayMs(recent.length) : now, ownerPid: process.pid, nonce: governorId(), status: "started",
3280
3459
  quotaKey: identity.quotaKey, resetMs: budget?.resetMs ?? null, accounted: false, imported: true, exceptional };
3281
- state.reservations[`reservation:${id}`] = { leaseId: id, intentId: id, costs: { core: 1, graphql: 0 }, actualCosts: null,
3460
+ state.controlReceipts.core = { leaseId: id, intentId: id, costs: { core: 1, graphql: 0 }, actualCosts: null,
3282
3461
  accountedCosts: { core: 0, graphql: 0 }, notBefore: now, status: "started", epochs: { core: budget?.epoch ?? null, graphql: null },
3283
- startedAt: now, completedAt: null, outcome: null };
3284
- const written = writeGovernorState(scope.path, state);
3285
- return written.ok ? { ok: true, value: { id, identity } } : written;
3462
+ startedAt: now, completedAt: null, outcome: null,
3463
+ receipt: {
3464
+ scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
3465
+ ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
3466
+ deadline: now + ACQUISITION_STARTED_DEADLINE_MS,
3467
+ allowance: { core: 1, graphql: 0 }, dispatches: [],
3468
+ } };
3469
+ const written = writeGovernorQuotaState(scope.path, state, now);
3470
+ return written.ok ? { ok: true, value: { id, identity,
3471
+ receiptCapability: { reservationId: `control:core`, controlResource: "core",
3472
+ leaseId: id, scopeHash: scope.hash,
3473
+ ownerNonce: state.controlReceipts.core.receipt.ownerNonce,
3474
+ generation: state.controlReceipts.core.receipt.generation,
3475
+ deadline: state.controlReceipts.core.receipt.deadline } } } : written;
3286
3476
  });
3287
3477
  }, { now });
3288
3478
  }
3289
3479
 
3290
- function settleIdentityControl(coordinator, control, stdout, now = Date.now()) {
3480
+ function settleIdentityControl(coordinator, control, stdout, now = Date.now(),
3481
+ { neverStarted = false } = {}) {
3291
3482
  return withIdentityRegistry(coordinator.root, (registry) => {
3292
3483
  const attempt = registry.attempts[control.id];
3293
3484
  if (!attempt) return { ok: false, reason: "stale" };
3294
3485
  const parsed = parseGhApiResponse(stdout ?? "");
3295
3486
  const evidence = pickRateLimit(parsed.headers);
3296
3487
  const proven = [200, 304].includes(parsed.status) && evidence?.resource === "core";
3297
- attempt.status = proven ? "finished" : "uncertain";
3298
- attempt.accounted = proven;
3488
+ attempt.status = proven || neverStarted ? "finished" : "uncertain";
3489
+ attempt.accounted = proven || neverStarted;
3299
3490
  if (evidence?.resource === "core") {
3300
3491
  attempt.resetMs = evidence.resetMs;
3301
3492
  if (evidence.remaining === 0) attempt.retryAt = Math.max(attempt.retryAt, evidence.resetMs + BUDGET_RESET_GRACE_MS);
@@ -3304,64 +3495,93 @@ function settleIdentityControl(coordinator, control, stdout, now = Date.now()) {
3304
3495
  return withGovernorLock(scope, () => {
3305
3496
  const loaded = readIdentityQuotaState(registry, scope, now);
3306
3497
  if (!loaded.ok) return loaded;
3307
- const reservation = loaded.value.reservations[`reservation:${control.id}`];
3498
+ const reservation = loaded.value.controlReceipts.core?.intentId === control.id
3499
+ ? loaded.value.controlReceipts.core : null;
3308
3500
  if (!reservation) return { ok: false, reason: "stale" };
3309
- if (proven) {
3310
- const cost = parsed.status === 304 ? 0 : 1;
3501
+ if (proven || neverStarted) {
3502
+ const cost = neverStarted || parsed.status === 304 ? 0 : 1;
3311
3503
  reservation.status = "completed";
3312
3504
  reservation.completedAt = now;
3313
3505
  reservation.outcome = "measured-success";
3314
3506
  reservation.actualCosts = { core: cost, graphql: 0 };
3315
3507
  reservation.accountedCosts = { core: cost, graphql: 0 };
3508
+ loaded.value.controlReceipts.core = null;
3509
+ } else if (!retireControlReceiptToDebt(loaded.value, "core", now)) {
3510
+ return { ok: false, reason: "accounting-overflow" };
3316
3511
  }
3317
- return writeGovernorState(scope.path, loaded.value);
3512
+ return writeGovernorQuotaState(scope.path, loaded.value, now);
3318
3513
  });
3319
3514
  }, { now });
3320
3515
  }
3321
3516
 
3517
+ const HTTP_PERMIT_FAILURE_HOLDS = Object.freeze({
3518
+ "transport-busy": "coordination",
3519
+ "throttle-paused": "secondary",
3520
+ });
3521
+
3522
+ function httpPermitFailure(reason) {
3523
+ const error = new Error(identityCoordinationMessage(reason));
3524
+ if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, reason)) error.coordinationReason = reason;
3525
+ return error;
3526
+ }
3527
+
3322
3528
  async function acquireIdentityHttpPermit(coordinator, {
3323
3529
  signal,
3324
3530
  now = Date.now,
3325
3531
  wait = (delay) => new Promise((resolve) => setTimeout(resolve, delay)),
3532
+ operationDeadline,
3326
3533
  } = {}) {
3327
3534
  const flushed = await coordinator.flushCompletions?.();
3328
- if (flushed && !flushed.ok) throw new Error(identityCoordinationMessage(flushed.reason));
3535
+ if (flushed && !flushed.ok) throw httpPermitFailure(flushed.reason);
3329
3536
  const identity = coordinator.current();
3330
3537
  if (!identity) throw new Error("Verified identity unavailable");
3331
3538
  const queuedAt = now();
3332
- const deadline = queuedAt + GH_TIMEOUT_MS;
3539
+ const deadline = Number.isFinite(operationDeadline)
3540
+ ? Math.min(operationDeadline, queuedAt + ACQUISITION_STARTED_DEADLINE_MS)
3541
+ : queuedAt + GH_TIMEOUT_MS;
3333
3542
  const nonce = governorId();
3334
3543
  let acquired = false;
3335
3544
  try {
3336
3545
  for (;;) {
3337
3546
  if (signal?.aborted) throw new Error("Request cancelled");
3338
3547
  if (coordinator.current()?.accessKey !== identity.accessKey) throw new Error("Credential changed before request start");
3339
- if (now() >= deadline) throw new Error(identityCoordinationMessage("transport-busy"));
3548
+ if (now() >= deadline) throw httpPermitFailure("transport-busy");
3340
3549
  const result = withIdentityRegistry(coordinator.root, (state) => {
3341
3550
  const migration = inspectLegacyMigration(coordinator.root, state, now());
3342
3551
  if (!migration.ok) return migration;
3343
- if (now() >= deadline) return { ok: false, reason: "transport-busy", retryAt: deadline };
3344
- const transport = state.hosts[identity.host] ??= { lastStartedAt: 0, cooldownUntil: 0, permit: null, waiters: [], throttle: emptyTransportThrottle() };
3552
+ const permitAt = now();
3553
+ if (permitAt >= deadline) return { ok: false, reason: "transport-busy", retryAt: deadline };
3554
+ const transport = state.hosts[identity.host] ??= emptyIdentityTransport();
3345
3555
  // Five consecutive throttles means the ladder has stopped being useful.
3346
3556
  // Waiting out the deadline is what produced the last one, so the pause
3347
3557
  // holds until a person retries or a primary reset opens a recovery.
3348
3558
  if (transport.throttle?.paused) {
3349
3559
  return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
3350
3560
  }
3351
- if (transport.permit && staleHttpPermit(transport.permit, now())) transport.permit = null;
3561
+ transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, permitAt));
3352
3562
  // Admission can happen during the mandatory start gap. Persist call
3353
3563
  // order once so later polling cannot overtake an earlier caller.
3354
- if (!transport.waiters.some((waiter) => waiter.nonce === nonce)) {
3564
+ let waiter = transport.waiters.find((item) => item.nonce === nonce);
3565
+ if (!waiter) {
3355
3566
  if (transport.waiters.length >= HTTP_MAX_WAITERS) return { ok: false, reason: "identity-capacity" };
3356
- transport.waiters.push({ pid: process.pid, nonce, queuedAt, deadline });
3567
+ waiter = { pid: process.pid, nonce, queuedAt: permitAt, deadline: Math.min(deadline, permitAt + GH_TIMEOUT_MS) };
3568
+ transport.waiters.push(waiter);
3569
+ } else if (waiter.deadline - permitAt <= GH_TIMEOUT_MS / 2 && waiter.deadline < deadline) {
3570
+ // Keep the existing registry's short waiter lease and FIFO position.
3571
+ // Renewing this lease cannot extend the admitted operation deadline
3572
+ // above, or the separate timeout of a child that actually starts.
3573
+ waiter.queuedAt = permitAt;
3574
+ waiter.deadline = Math.min(deadline, waiter.queuedAt + GH_TIMEOUT_MS);
3357
3575
  }
3358
3576
  const retryAt = Math.max(transport.cooldownUntil, transport.lastStartedAt + HTTP_START_GAP_MS);
3359
- if (transport.permit || transport.waiters[0].nonce !== nonce || retryAt > now()) {
3360
- return { ok: false, reason: "transport-busy", retryAt: Math.max(retryAt, now() + 50) };
3577
+ if (transport.permits.length >= HTTP_MAX_PERMITS ||
3578
+ transport.permits.some((permit) => permit.kind === "bootstrap") ||
3579
+ transport.waiters[0].nonce !== nonce || retryAt > permitAt) {
3580
+ return { ok: false, reason: "transport-busy", retryAt: Math.max(retryAt, permitAt + 50) };
3361
3581
  }
3362
3582
  transport.waiters.shift();
3363
- transport.permit = { pid: process.pid, nonce, startedAt: now() };
3364
- transport.lastStartedAt = now();
3583
+ transport.permits.push({ pid: process.pid, nonce, startedAt: permitAt, kind: "request", bootId: null });
3584
+ transport.lastStartedAt = permitAt;
3365
3585
  return { ok: true, value: { nonce, host: identity.host } };
3366
3586
  }, { now: now() });
3367
3587
  if (result.ok) { acquired = true; return result.value; }
@@ -3370,7 +3590,7 @@ async function acquireIdentityHttpPermit(coordinator, {
3370
3590
  // retryAt, so give the rest the poll interval the loop already uses.
3371
3591
  const retryAt = result.retryAt ?? now() + 50;
3372
3592
  if ((result.reason !== "transport-busy" && !retryableCoordination(result.reason)) ||
3373
- now() >= deadline || retryAt >= deadline) throw new Error(identityCoordinationMessage(result.reason));
3593
+ now() >= deadline || retryAt >= deadline) throw httpPermitFailure(result.reason);
3374
3594
  await wait(Math.max(1, Math.min(50, retryAt - now())));
3375
3595
  }
3376
3596
  } finally {
@@ -3454,6 +3674,10 @@ function transportCooldownDeadline({ retryAfter, status, secondary, at, attempts
3454
3674
  return status === 429 || secondary === true ? at + throttleLadderMs(attempts) : null;
3455
3675
  }
3456
3676
 
3677
+ function emptyIdentityTransport() {
3678
+ return { lastStartedAt: 0, cooldownUntil: 0, permits: [], waiters: [], throttle: emptyTransportThrottle() };
3679
+ }
3680
+
3457
3681
  function emptyTransportThrottle() {
3458
3682
  return { attempts: 0, lastAt: 0, paused: false };
3459
3683
  }
@@ -3481,10 +3705,21 @@ function clearTransportThrottle(transport) {
3481
3705
  }
3482
3706
  }
3483
3707
 
3708
+ // Completion retries keep their original response time and become no-ops
3709
+ // after a successful durable write. Weak keys do not accumulate tombstones.
3710
+ const identityPermitCompletions = new WeakMap();
3484
3711
  function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date.now()) {
3485
- return withIdentityRegistry(coordinator.root, (state) => {
3712
+ let completion = identityPermitCompletions.get(permit);
3713
+ if (!completion) {
3714
+ completion = { at: now, done: false };
3715
+ identityPermitCompletions.set(permit, completion);
3716
+ }
3717
+ if (completion.done) return { ok: true };
3718
+ now = completion.at;
3719
+ const result = withIdentityRegistry(coordinator.root, (state) => {
3486
3720
  const transport = state.hosts[permit.host];
3487
- if (transport?.permit?.nonce === permit.nonce) transport.permit = null;
3721
+ const held = transport?.permits.find((item) => item.nonce === permit.nonce);
3722
+ if (transport) transport.permits = transport.permits.filter((item) => item.nonce !== permit.nonce);
3488
3723
  if (transport && error) {
3489
3724
  const response = parseGhApiResponse(error.stdout ?? "");
3490
3725
  const verdict = classifyThrottle({
@@ -3503,11 +3738,14 @@ function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date
3503
3738
  at: now,
3504
3739
  });
3505
3740
  }
3506
- } else if (transport && !error) {
3741
+ } else if (transport && !error && held && held.startedAt > transport.throttle.lastAt &&
3742
+ held.startedAt >= transport.cooldownUntil) {
3507
3743
  clearTransportThrottle(transport);
3508
3744
  }
3509
3745
  return { ok: true };
3510
3746
  });
3747
+ if (result.ok) completion.done = true;
3748
+ return result;
3511
3749
  }
3512
3750
 
3513
3751
  // The explicit retry a paused transport waits for. Clears the ladder and the
@@ -3558,11 +3796,11 @@ const GOVERNOR_SCOPE_VERSION = 1;
3558
3796
  // 5: scheduling fairness is persisted, because the starvation it prevents
3559
3797
  // happens *between* planning passes -- each manual refresh is its own pass, so
3560
3798
  // a counter that lives only inside one cannot see a run of them.
3561
- const GOVERNOR_STATE_VERSION = 6;
3799
+ const GOVERNOR_STATE_VERSION = 7;
3562
3800
  // Readable-as-evidence, never written: the shapes a still-running older pane
3563
3801
  // holds. Recognising that such a pane owns a live lease is what the restart
3564
3802
  // boundary depends on.
3565
- const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 5, 4, 3, 2];
3803
+ const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 6, 5, 4, 3, 2];
3566
3804
  // The first version whose file is already in the per-resource observer shape.
3567
3805
  // Anything below it is adapted before it can be read; see readLegacyGovernorState.
3568
3806
  const GOVERNOR_OBSERVER_SPLIT_VERSION = 4;
@@ -3582,6 +3820,8 @@ const MANUAL_GRANT_STREAK_LIMIT = 3;
3582
3820
  const GOVERNOR_MAX_LEASES = 128;
3583
3821
  const GOVERNOR_MAX_INTENTS = 512;
3584
3822
  const GOVERNOR_MAX_RESERVATIONS = 512;
3823
+ const GOVERNOR_MAX_DEBT_GROUPS = 128;
3824
+ const GOVERNOR_MAX_LEDGER_BYTES = 2 * 1024 * 1024;
3585
3825
  const GOVERNOR_LOCK_WAIT_MS = 250;
3586
3826
  // How old an unreadable lock record may be before it counts as abandoned. A
3587
3827
  // creator owns the path from open("wx") until its owner record is written, a
@@ -3697,6 +3937,7 @@ function currentGovernorScope(scope) {
3697
3937
  function emptyGovernorState() {
3698
3938
  return {
3699
3939
  version: GOVERNOR_STATE_VERSION,
3940
+ revision: 0,
3700
3941
  epochs: { core: null, graphql: null },
3701
3942
  budgets: {},
3702
3943
  fairness: { manualStreak: 0 },
@@ -3712,6 +3953,14 @@ function emptyGovernorState() {
3712
3953
  leases: {},
3713
3954
  intents: {},
3714
3955
  reservations: {},
3956
+ controlReceipts: { core: null, graphql: null },
3957
+ debt: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, {
3958
+ unresolvedUnits: 0, unresolvedCount: 0, quiescentUnits: 0, quiescentCount: 0,
3959
+ revision: 0,
3960
+ }])),
3961
+ debtGroups: {},
3962
+ ownerGenerations: {},
3963
+ importMarkers: {},
3715
3964
  manualProbe: null,
3716
3965
  };
3717
3966
  }
@@ -3804,10 +4053,11 @@ function normalizeGovernorIntent(raw, nowMs) {
3804
4053
  return { ...raw, costs };
3805
4054
  }
3806
4055
 
3807
- function normalizeGovernorReservation(raw, nowMs) {
4056
+ function normalizeGovernorReservation(raw, nowMs, { legacy = false } = {}) {
3808
4057
  if (!exactKeys(raw, [
3809
4058
  "leaseId", "intentId", "costs", "actualCosts", "accountedCosts", "notBefore", "status", "epochs",
3810
4059
  "startedAt", "completedAt", "outcome",
4060
+ ...(!legacy ? ["receipt"] : []),
3811
4061
  ])) return null;
3812
4062
  const costs = exactResourceCosts(raw.costs);
3813
4063
  const actualCosts = raw.actualCosts === null ? null : exactResourceCosts(raw.actualCosts);
@@ -3829,13 +4079,46 @@ function normalizeGovernorReservation(raw, nowMs) {
3829
4079
  ) return null;
3830
4080
  if (actualCosts && RATE_RESOURCES.some((resource) => actualCosts[resource] > costs[resource])) return null;
3831
4081
  if (RATE_RESOURCES.some((resource) => accountedCosts[resource] > (actualCosts?.[resource] ?? costs[resource]))) return null;
4082
+ if (!legacy && raw.receipt !== null) {
4083
+ const receipt = raw.receipt;
4084
+ if (!exactKeys(receipt, [
4085
+ "scopeHash", "ownerNonce", "generation", "ownerPid", "ownerBirth", "bootId",
4086
+ "deadline", "allowance", "dispatches",
4087
+ ]) || typeof receipt.scopeHash !== "string" || !/^[0-9a-f]{64}$/.test(receipt.scopeHash) ||
4088
+ !validGovernorId(receipt.ownerNonce) || !validGovernorId(receipt.generation) ||
4089
+ !Number.isSafeInteger(receipt.ownerPid) || receipt.ownerPid < 1 ||
4090
+ receipt.ownerBirth !== null && (typeof receipt.ownerBirth !== "string" || receipt.ownerBirth.length > 128) ||
4091
+ receipt.bootId !== null && (typeof receipt.bootId !== "string" || receipt.bootId.length > 128) ||
4092
+ finiteTimestamp(receipt.deadline, nowMs) === undefined ||
4093
+ !exactKeys(receipt.allowance, RATE_RESOURCES) ||
4094
+ RATE_RESOURCES.some((resource) => receipt.allowance[resource] !== costs[resource]) ||
4095
+ !Array.isArray(receipt.dispatches) || receipt.dispatches.length > 32) return null;
4096
+ const issued = { core: 0, graphql: 0 };
4097
+ for (const [index, dispatch] of receipt.dispatches.entries()) {
4098
+ if (!exactKeys(dispatch, ["sequence", "operation", "costs", "issuedAt", "terminalAt", "childPid", "childBirth", "neverStarted"]) ||
4099
+ dispatch.sequence !== index + 1 || !operationCost(dispatch.operation) ||
4100
+ !exactKeys(dispatch.costs, RATE_RESOURCES) ||
4101
+ RATE_RESOURCES.some((resource) => dispatch.costs[resource] !== operationCost(dispatch.operation)[resource]) ||
4102
+ finiteTimestamp(dispatch.issuedAt, nowMs) === undefined ||
4103
+ finiteTimestamp(dispatch.terminalAt, nowMs, { nullable: true }) === undefined ||
4104
+ dispatch.childPid !== null && (!Number.isSafeInteger(dispatch.childPid) || dispatch.childPid < 1) ||
4105
+ dispatch.childBirth !== null && (typeof dispatch.childBirth !== "string" || dispatch.childBirth.length > 128) ||
4106
+ typeof dispatch.neverStarted !== "boolean" ||
4107
+ dispatch.neverStarted && (dispatch.childPid !== null || dispatch.terminalAt === null)) return null;
4108
+ for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
4109
+ }
4110
+ if (RATE_RESOURCES.some((resource) => !Number.isSafeInteger(issued[resource]) ||
4111
+ issued[resource] > receipt.allowance[resource])) return null;
4112
+ } else if (!legacy && raw.status === "started") {
4113
+ return null;
4114
+ }
3832
4115
  return { ...raw, costs, actualCosts, accountedCosts, epochs };
3833
4116
  }
3834
4117
 
3835
4118
  function normalizeResourceObserver(raw, nowMs) {
3836
4119
  if (!exactKeys(raw, ["etag", "outcome", "at", "nextAt"])) return null;
3837
4120
  if (
3838
- raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0) ||
4121
+ raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0 || raw.etag.length > 512) ||
3839
4122
  !GOVERNOR_PROBE_STATUSES.has(raw.outcome) ||
3840
4123
  finiteTimestamp(raw.at, nowMs) === undefined || raw.at > nowMs ||
3841
4124
  finiteTimestamp(raw.nextAt, nowMs) === undefined
@@ -3843,10 +4126,11 @@ function normalizeResourceObserver(raw, nowMs) {
3843
4126
  return { ...raw };
3844
4127
  }
3845
4128
 
3846
- function normalizeProbeClaim(raw, nowMs) {
4129
+ function normalizeProbeClaim(raw, nowMs, { legacy = false } = {}) {
3847
4130
  if (raw === null) return null;
3848
4131
  if (!exactKeys(raw, [
3849
4132
  "ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds",
4133
+ ...(!legacy ? ["debtSnapshot"] : []),
3850
4134
  ])) {
3851
4135
  return undefined;
3852
4136
  }
@@ -3858,6 +4142,12 @@ function normalizeProbeClaim(raw, nowMs) {
3858
4142
  !Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
3859
4143
  raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
3860
4144
  ) return undefined;
4145
+ const snapshot = raw.debtSnapshot;
4146
+ if (!legacy && (!exactKeys(snapshot, ["units", "count", "barrierAt", "revision"]) ||
4147
+ !Number.isSafeInteger(snapshot.units) || snapshot.units < 0 ||
4148
+ !Number.isSafeInteger(snapshot.count) || snapshot.count < 0 ||
4149
+ finiteTimestamp(snapshot.barrierAt, nowMs) === undefined ||
4150
+ !Number.isSafeInteger(snapshot.revision) || snapshot.revision < 0)) return undefined;
3861
4151
  return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
3862
4152
  }
3863
4153
 
@@ -3878,9 +4168,11 @@ function normalizeManualProbe(raw, nowMs) {
3878
4168
  // never publishes what it reads. Every other caller takes the default, so a
3879
4169
  // version this build does not write is unreadable and fails closed.
3880
4170
  function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVERNOR_STATE_VERSION } = {}) {
4171
+ const legacyShape = acceptVersion < 7;
3881
4172
  if (!exactKeys(raw, [
3882
4173
  "version", "epochs", "budgets", "fairness", "observers", "probeClaims", "leases",
3883
4174
  "intents", "reservations", "manualProbe",
4175
+ ...(!legacyShape ? ["revision", "controlReceipts", "debt", "debtGroups", "ownerGenerations", "importMarkers"] : []),
3884
4176
  ]) || raw.version !== acceptVersion) return null;
3885
4177
  if (!exactKeys(raw.fairness, ["manualStreak"]) ||
3886
4178
  !Number.isSafeInteger(raw.fairness.manualStreak) || raw.fairness.manualStreak < 0) return null;
@@ -3894,6 +4186,75 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3894
4186
  ) return null;
3895
4187
 
3896
4188
  const state = emptyGovernorState();
4189
+ state.version = acceptVersion;
4190
+ if (legacyShape) {
4191
+ delete state.revision;
4192
+ delete state.controlReceipts;
4193
+ delete state.debt;
4194
+ delete state.debtGroups;
4195
+ delete state.ownerGenerations;
4196
+ delete state.importMarkers;
4197
+ } else {
4198
+ if (!Number.isSafeInteger(raw.revision) || raw.revision < 0) return null;
4199
+ state.revision = raw.revision;
4200
+ if (!exactKeys(raw.controlReceipts, RATE_RESOURCES) || !exactKeys(raw.debt, RATE_RESOURCES) ||
4201
+ !isRecord(raw.debtGroups) || Object.keys(raw.debtGroups).length > GOVERNOR_MAX_DEBT_GROUPS ||
4202
+ !isRecord(raw.ownerGenerations) ||
4203
+ Object.keys(raw.ownerGenerations).length > GOVERNOR_MAX_LEASES + GOVERNOR_MAX_RESERVATIONS ||
4204
+ !isRecord(raw.importMarkers) || Object.keys(raw.importMarkers).length > IDENTITY_MAX_ATTEMPTS) return null;
4205
+ for (const [id, marker] of Object.entries(raw.importMarkers)) {
4206
+ if (!validGovernorId(id) || marker !== true) return null;
4207
+ state.importMarkers[id] = true;
4208
+ }
4209
+ for (const [leaseId, generation] of Object.entries(raw.ownerGenerations)) {
4210
+ if (!validGovernorId(leaseId) || !validGovernorId(generation)) return null;
4211
+ state.ownerGenerations[leaseId] = generation;
4212
+ }
4213
+ for (const resource of RATE_RESOURCES) {
4214
+ const debt = raw.debt[resource];
4215
+ if (!exactKeys(debt, ["unresolvedUnits", "unresolvedCount", "quiescentUnits", "quiescentCount", "revision"]) ||
4216
+ Object.values(debt).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
4217
+ state.debt[resource] = { ...debt };
4218
+ // Control receipts are normalized independently from the 512 data slots.
4219
+ if (raw.controlReceipts[resource] !== null) {
4220
+ const control = normalizeGovernorReservation(raw.controlReceipts[resource], nowMs);
4221
+ if (!control) return null;
4222
+ state.controlReceipts[resource] = control;
4223
+ }
4224
+ }
4225
+ for (const [id, group] of Object.entries(raw.debtGroups)) {
4226
+ if (!validGovernorId(id) || !exactKeys(group, [
4227
+ "nonce", "ownerNonce", "ownerPid", "ownerBirth", "bootId", "unknownChild", "sealed",
4228
+ "quiescent", "units", "counts", "at", "barrierAt", "barrierRevisions",
4229
+ ]) || !validGovernorId(group.nonce) ||
4230
+ group.ownerNonce !== null && !validGovernorId(group.ownerNonce) ||
4231
+ group.ownerPid !== null && (!Number.isSafeInteger(group.ownerPid) || group.ownerPid < 1) ||
4232
+ group.ownerBirth !== null && (typeof group.ownerBirth !== "string" || group.ownerBirth.length > 128) ||
4233
+ group.bootId !== null && (typeof group.bootId !== "string" || group.bootId.length > 128) ||
4234
+ typeof group.unknownChild !== "boolean" || typeof group.sealed !== "boolean" ||
4235
+ typeof group.quiescent !== "boolean" || !exactKeys(group.units, RATE_RESOURCES) ||
4236
+ !exactKeys(group.counts, RATE_RESOURCES) ||
4237
+ [...Object.values(group.units), ...Object.values(group.counts)].some((value) =>
4238
+ !Number.isSafeInteger(value) || value < 0) ||
4239
+ finiteTimestamp(group.at, nowMs) === undefined ||
4240
+ finiteTimestamp(group.barrierAt, nowMs, { nullable: true }) === undefined ||
4241
+ !exactKeys(group.barrierRevisions, RATE_RESOURCES) ||
4242
+ Object.values(group.barrierRevisions).some((value) =>
4243
+ !Number.isSafeInteger(value) || value < 0)) return null;
4244
+ state.debtGroups[id] = structuredClone(group);
4245
+ }
4246
+ for (const resource of RATE_RESOURCES) {
4247
+ const totals = { unresolvedUnits: 0, unresolvedCount: 0,
4248
+ quiescentUnits: 0, quiescentCount: 0 };
4249
+ for (const group of Object.values(state.debtGroups)) {
4250
+ const prefix = group.quiescent ? "quiescent" : "unresolved";
4251
+ totals[`${prefix}Units`] += group.units[resource];
4252
+ totals[`${prefix}Count`] += group.counts[resource];
4253
+ }
4254
+ if (Object.entries(totals).some(([field, amount]) =>
4255
+ !Number.isSafeInteger(amount) || amount !== state.debt[resource][field])) return null;
4256
+ }
4257
+ }
3897
4258
  state.epochs = { ...raw.epochs };
3898
4259
  state.fairness = { ...raw.fairness };
3899
4260
  for (const resource of RATE_RESOURCES) {
@@ -3922,7 +4283,7 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3922
4283
  if (Object.keys(state.intents).length > GOVERNOR_MAX_INTENTS) return null;
3923
4284
  for (const [id, rawReservation] of Object.entries(raw.reservations)) {
3924
4285
  if (!id.startsWith("reservation:") || !validGovernorId(id.slice(12))) return null;
3925
- const reservation = normalizeGovernorReservation(rawReservation, nowMs);
4286
+ const reservation = normalizeGovernorReservation(rawReservation, nowMs, { legacy: legacyShape });
3926
4287
  if (!reservation || id !== `reservation:${reservation.intentId}`) return null;
3927
4288
  if (
3928
4289
  !prune || reservation.status === "started" || reservation.status === "completed" ||
@@ -3930,8 +4291,9 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3930
4291
  ) state.reservations[id] = reservation;
3931
4292
  }
3932
4293
  if (Object.keys(state.reservations).length > GOVERNOR_MAX_RESERVATIONS) return null;
4294
+ if (prune && !legacyShape) retireOwnerGenerations(state);
3933
4295
  for (const resource of RATE_RESOURCES) {
3934
- const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs);
4296
+ const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs, { legacy: legacyShape });
3935
4297
  if (claim === undefined) return null;
3936
4298
  state.probeClaims[resource] = claim?.leaseUntil > nowMs ? claim : null;
3937
4299
  }
@@ -3940,6 +4302,17 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
3940
4302
  return state;
3941
4303
  }
3942
4304
 
4305
+ function retireOwnerGenerations(state) {
4306
+ const detailedOwners = new Set(Object.values(state.reservations)
4307
+ .filter((reservation) => reservation.receipt !== null)
4308
+ .map((reservation) => reservation.leaseId));
4309
+ for (const leaseId of Object.keys(state.ownerGenerations)) {
4310
+ if (!state.leases[leaseId] && !detailedOwners.has(leaseId)) {
4311
+ delete state.ownerGenerations[leaseId];
4312
+ }
4313
+ }
4314
+ }
4315
+
3943
4316
  // Reads a still-running older pane's file as evidence. A version number alone
3944
4317
  // is not enough to identify a document: versions 2 and 3 share a *shape* that
3945
4318
  // version 4 replaced, so accepting their version against the current key set
@@ -4048,8 +4421,296 @@ function migrateGovernorState(raw, nowMs) {
4048
4421
  : adaptPreSplitGovernorShape(raw);
4049
4422
  if (!shaped) return null;
4050
4423
  if (!isRecord(shaped.fairness)) shaped.fairness = { manualStreak: 0 };
4051
- shaped.version = GOVERNOR_STATE_VERSION;
4052
- return normalizeGovernorState(dropSupersededTabCosts(shaped), nowMs);
4424
+ shaped.version = 6;
4425
+ const legacy = normalizeGovernorState(dropSupersededTabCosts(shaped), nowMs, {
4426
+ prune: false,
4427
+ acceptVersion: 6,
4428
+ });
4429
+ return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
4430
+ }
4431
+
4432
+ function migrateLegacyGovernorReceipts(legacy, nowMs) {
4433
+ const state = { ...legacy, version: GOVERNOR_STATE_VERSION,
4434
+ revision: 0,
4435
+ controlReceipts: { core: null, graphql: null },
4436
+ debt: emptyGovernorState().debt,
4437
+ debtGroups: {},
4438
+ ownerGenerations: {},
4439
+ importMarkers: {},
4440
+ reservations: {},
4441
+ };
4442
+ state.probeClaims = { core: null, graphql: null };
4443
+ for (const resource of RATE_RESOURCES) {
4444
+ if (legacy.probeClaims[resource]) {
4445
+ state.observers[resource] = { ...state.observers[resource], outcome: "idle", nextAt: nowMs };
4446
+ }
4447
+ }
4448
+ const units = { core: 0, graphql: 0 };
4449
+ const counts = { core: 0, graphql: 0 };
4450
+ for (const reservation of Object.values(legacy.reservations)) {
4451
+ for (const resource of RATE_RESOURCES) {
4452
+ const charge = reservationCost(reservation, resource, legacy.leases, nowMs);
4453
+ if (!Number.isSafeInteger(charge) || !Number.isSafeInteger(units[resource] + charge)) return null;
4454
+ units[resource] += charge;
4455
+ if (charge > 0) counts[resource] += 1;
4456
+ }
4457
+ }
4458
+ if (RATE_RESOURCES.some((resource) => units[resource] > 0)) {
4459
+ const nonce = governorId();
4460
+ state.debtGroups[nonce] = {
4461
+ nonce, ownerNonce: null, ownerPid: null, ownerBirth: null, bootId: coordinationBootId(),
4462
+ unknownChild: true, sealed: true, quiescent: false,
4463
+ units, counts, at: nowMs, barrierAt: null,
4464
+ barrierRevisions: { core: 0, graphql: 0 },
4465
+ };
4466
+ for (const resource of RATE_RESOURCES) {
4467
+ state.debt[resource].unresolvedUnits = units[resource];
4468
+ state.debt[resource].unresolvedCount = counts[resource];
4469
+ state.debt[resource].revision = 1;
4470
+ }
4471
+ }
4472
+ return normalizeGovernorState(state, nowMs);
4473
+ }
4474
+
4475
+ const UNKNOWN_DEBT_GROUP_ID = "ffffffff-ffff-4fff-8fff-ffffffffffff";
4476
+
4477
+ let cachedCoordinationBootId;
4478
+ let cachedCoordinationProcessBirth;
4479
+ function coordinationBootId() {
4480
+ if (cachedCoordinationBootId !== undefined) return cachedCoordinationBootId;
4481
+ try {
4482
+ const value = process.platform === "linux"
4483
+ ? readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()
4484
+ : process.platform === "darwin"
4485
+ ? execFileSync("/usr/sbin/sysctl", ["-n", "kern.boottime"], { encoding: "utf8", timeout: 1000 }).trim()
4486
+ : "";
4487
+ cachedCoordinationBootId = value.length > 0 && value.length <= 128 ? value : null;
4488
+ } catch {
4489
+ cachedCoordinationBootId = null;
4490
+ }
4491
+ return cachedCoordinationBootId;
4492
+ }
4493
+
4494
+ function coordinationProcessBirth(pid = process.pid) {
4495
+ if (pid === process.pid && cachedCoordinationProcessBirth !== undefined) {
4496
+ return cachedCoordinationProcessBirth;
4497
+ }
4498
+ let value = null;
4499
+ try {
4500
+ if (process.platform === "linux") {
4501
+ const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
4502
+ const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/);
4503
+ value = fields[19] ?? null; // starttime is field 22, after pid and comm.
4504
+ } else if (process.platform === "darwin") {
4505
+ value = execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], {
4506
+ encoding: "utf8", timeout: 1000,
4507
+ }).trim() || null;
4508
+ }
4509
+ } catch { /* an unavailable birth identity is explicitly unknown */ }
4510
+ if (value && value.length > 128) value = null;
4511
+ if (pid === process.pid) cachedCoordinationProcessBirth = value;
4512
+ return value;
4513
+ }
4514
+
4515
+ function debtGroupTarget(state, receipt, groupId = null) {
4516
+ const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
4517
+ const count = Object.keys(state.debtGroups).length;
4518
+ const owned = groupId ? state.debtGroups[groupId] : null;
4519
+ if (owned && !owned.quiescent && owned.bootId === incomingBootId &&
4520
+ (!receipt.receipt || owned.ownerNonce === null ||
4521
+ owned.ownerNonce === receipt.receipt.ownerNonce)) return groupId;
4522
+ if (groupId && !owned && count < GOVERNOR_MAX_DEBT_GROUPS - 1) return groupId;
4523
+ const shared = Object.entries(state.debtGroups).find(([, group]) =>
4524
+ !group.quiescent && group.ownerNonce === null && group.unknownChild &&
4525
+ group.bootId === incomingBootId);
4526
+ if (shared) return shared[0];
4527
+ if (count >= GOVERNOR_MAX_DEBT_GROUPS) return null;
4528
+ return state.debtGroups[UNKNOWN_DEBT_GROUP_ID] ? governorId() : UNKNOWN_DEBT_GROUP_ID;
4529
+ }
4530
+
4531
+ function transferReceiptToDebt(state, receipt, nowMs, groupId = null) {
4532
+ const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
4533
+ reservationCost(receipt, resource, state.leases, nowMs)]));
4534
+ if (RATE_RESOURCES.every((resource) => units[resource] === 0)) return true;
4535
+ const id = debtGroupTarget(state, receipt, groupId);
4536
+ if (id === null) return false;
4537
+ const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
4538
+ const dedicated = id === groupId;
4539
+ const group = state.debtGroups[id] ?? {
4540
+ nonce: governorId(), ownerNonce: null, ownerPid: null, ownerBirth: null,
4541
+ bootId: incomingBootId,
4542
+ unknownChild: !dedicated || !receipt.receipt, sealed: true, quiescent: false,
4543
+ units: { core: 0, graphql: 0 }, counts: { core: 0, graphql: 0 },
4544
+ at: nowMs, barrierAt: null,
4545
+ barrierRevisions: { core: 0, graphql: 0 },
4546
+ };
4547
+ if (group.quiescent) return false;
4548
+ if (receipt.receipt && dedicated) {
4549
+ if (group.ownerNonce !== null && group.ownerNonce !== receipt.receipt.ownerNonce) return false;
4550
+ group.ownerNonce = receipt.receipt.ownerNonce;
4551
+ group.ownerPid = receipt.receipt.ownerPid;
4552
+ group.ownerBirth = receipt.receipt.ownerBirth;
4553
+ group.bootId = receipt.receipt.bootId;
4554
+ group.unknownChild ||= receipt.receipt.dispatches.some((dispatch) =>
4555
+ dispatch.terminalAt === null);
4556
+ }
4557
+ for (const resource of RATE_RESOURCES) {
4558
+ const amount = units[resource];
4559
+ if (!Number.isSafeInteger(amount) || amount < 0 ||
4560
+ !Number.isSafeInteger(state.debt[resource].unresolvedUnits + amount) ||
4561
+ !Number.isSafeInteger(group.units[resource] + amount) ||
4562
+ !Number.isSafeInteger(state.debt[resource].unresolvedCount + (amount > 0 ? 1 : 0)) ||
4563
+ !Number.isSafeInteger(state.debt[resource].revision + 1) ||
4564
+ !Number.isSafeInteger(group.counts[resource] + (amount > 0 ? 1 : 0))) return false;
4565
+ }
4566
+ for (const resource of RATE_RESOURCES) {
4567
+ const amount = units[resource];
4568
+ state.debt[resource].unresolvedUnits += amount;
4569
+ state.debt[resource].unresolvedCount += amount > 0 ? 1 : 0;
4570
+ state.debt[resource].revision += 1;
4571
+ group.units[resource] += amount;
4572
+ group.counts[resource] += amount > 0 ? 1 : 0;
4573
+ }
4574
+ // The newest charge, not the group's creation, starts the window after
4575
+ // which reconcileBootDebt may treat the group as settled.
4576
+ group.at = Math.max(group.at, nowMs);
4577
+ state.debtGroups[id] = group;
4578
+ return true;
4579
+ }
4580
+
4581
+ function makeDebtGroupQuiescent(state, group, at) {
4582
+ for (const resource of RATE_RESOURCES) {
4583
+ const debt = state.debt[resource];
4584
+ if (debt.unresolvedUnits < group.units[resource] || debt.unresolvedCount < group.counts[resource] ||
4585
+ !Number.isSafeInteger(debt.quiescentUnits + group.units[resource]) ||
4586
+ !Number.isSafeInteger(debt.quiescentCount + group.counts[resource]) ||
4587
+ !Number.isSafeInteger(debt.revision + 1)) return false;
4588
+ }
4589
+ for (const resource of RATE_RESOURCES) {
4590
+ const debt = state.debt[resource];
4591
+ debt.unresolvedUnits -= group.units[resource];
4592
+ debt.unresolvedCount -= group.counts[resource];
4593
+ debt.quiescentUnits += group.units[resource];
4594
+ debt.quiescentCount += group.counts[resource];
4595
+ debt.revision += 1;
4596
+ }
4597
+ group.quiescent = true;
4598
+ group.barrierAt = at;
4599
+ group.barrierRevisions = Object.fromEntries(RATE_RESOURCES.map((resource) => [
4600
+ resource, state.debt[resource].revision,
4601
+ ]));
4602
+ return true;
4603
+ }
4604
+
4605
+ function retireControlReceiptToDebt(state, resource, at) {
4606
+ const control = state.controlReceipts[resource];
4607
+ if (!control) return true;
4608
+ // A control receipt without an issued dispatch cannot have started HTTP.
4609
+ // Removing it also fences a late owner before it can issue a child.
4610
+ if (control.receipt.dispatches.length === 0) {
4611
+ state.controlReceipts[resource] = null;
4612
+ return true;
4613
+ }
4614
+ const generation = control.receipt.generation;
4615
+ if (!transferReceiptToDebt(state, control, at, generation)) return false;
4616
+ const group = state.debtGroups[generation];
4617
+ if (group && !group.unknownChild && group.ownerPid !== null &&
4618
+ group.ownerBirth !== null && group.bootId !== null &&
4619
+ !makeDebtGroupQuiescent(state, group, at)) return false;
4620
+ state.controlReceipts[resource] = null;
4621
+ return true;
4622
+ }
4623
+
4624
+ // A reboot proves an unknown owner's children are gone. So does age: a group's
4625
+ // `at` is its newest charge, and once a full window has elapsed since then
4626
+ // GitHub has charged every covered request to a window that is already over.
4627
+ function reconcileBootDebt(state, at) {
4628
+ const bootId = coordinationBootId();
4629
+ let changed = false;
4630
+ for (const group of Object.values(state.debtGroups)) {
4631
+ if (group.quiescent) continue;
4632
+ const priorBoot = bootId !== null && group.bootId !== null && group.bootId !== bootId;
4633
+ const outlivedWindow = group.sealed && at - group.at >= IDENTITY_UNCERTAIN_MAX_MS;
4634
+ if (!priorBoot && !outlivedWindow) continue;
4635
+ if (!makeDebtGroupQuiescent(state, group, at)) {
4636
+ return { ok: false, reason: "accounting-overflow" };
4637
+ }
4638
+ changed = true;
4639
+ }
4640
+ return { ok: true, changed };
4641
+ }
4642
+
4643
+ function compactGovernorReceipts(state, nowMs) {
4644
+ const sealed = new Set();
4645
+ const full = Object.keys(state.reservations).length >= GOVERNOR_MAX_RESERVATIONS;
4646
+ let changed = false;
4647
+ const dueGenerations = new Set(Object.values(state.reservations)
4648
+ .filter((reservation) => reservation.receipt && (
4649
+ reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
4650
+ full && reservation.status === "completed"))
4651
+ .map((reservation) => reservation.receipt.generation));
4652
+ // A full debt journal must not prevent a claimed observer from retiring
4653
+ // older quiescent groups. Keep the detailed receipts until space is proven.
4654
+ const candidates = Object.values(state.reservations).filter((reservation) => {
4655
+ const groupId = reservation.receipt?.generation ?? null;
4656
+ return (full && reservation.status === "completed" && !reservation.receipt ||
4657
+ groupId && dueGenerations.has(groupId) &&
4658
+ !(reservation.status === "started" && reservation.receipt.dispatches.length === 0)) &&
4659
+ RATE_RESOURCES.some((resource) =>
4660
+ reservationCost(reservation, resource, state.leases, nowMs) > 0);
4661
+ });
4662
+ if (candidates.length > 0) {
4663
+ const trial = structuredClone(state);
4664
+ for (const reservation of candidates) {
4665
+ const groupId = reservation.receipt?.generation ?? null;
4666
+ if (debtGroupTarget(trial, reservation, groupId) === null) return { ok: true, changed: false };
4667
+ if (!transferReceiptToDebt(trial, reservation, nowMs, groupId)) {
4668
+ return { ok: false, reason: "accounting-overflow" };
4669
+ }
4670
+ }
4671
+ }
4672
+ if (full) {
4673
+ for (const [id, reservation] of Object.entries(state.reservations)) {
4674
+ if (reservation.status !== "completed" || reservation.receipt) continue;
4675
+ if (!transferReceiptToDebt(state, reservation, nowMs)) {
4676
+ return { ok: false, reason: "accounting-overflow" };
4677
+ }
4678
+ delete state.reservations[id];
4679
+ changed = true;
4680
+ }
4681
+ }
4682
+ for (const reservation of Object.values(state.reservations)) {
4683
+ if (reservation.receipt && (
4684
+ reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
4685
+ full && reservation.status === "completed")) {
4686
+ sealed.add(reservation.receipt.generation);
4687
+ }
4688
+ }
4689
+ if (sealed.size === 0) {
4690
+ if (changed) retireOwnerGenerations(state);
4691
+ return { ok: true, changed };
4692
+ }
4693
+ // Seal the complete owner generation. A late completion of one receipt cannot
4694
+ // make another already-authorized child in that generation disappear.
4695
+ for (const generation of sealed) {
4696
+ for (const [id, reservation] of Object.entries(state.reservations)) {
4697
+ if (reservation.receipt?.generation !== generation) continue;
4698
+ if (!(reservation.status === "started" && reservation.receipt.dispatches.length === 0) &&
4699
+ !transferReceiptToDebt(state, reservation, nowMs, generation)) {
4700
+ return { ok: false, reason: "accounting-overflow" };
4701
+ }
4702
+ delete state.reservations[id];
4703
+ state.ownerGenerations[reservation.leaseId] = governorId();
4704
+ }
4705
+ const group = state.debtGroups[generation];
4706
+ if (group && !group.unknownChild && group.ownerPid !== null &&
4707
+ group.ownerBirth !== null && group.bootId !== null &&
4708
+ !makeDebtGroupQuiescent(state, group, nowMs)) {
4709
+ return { ok: false, reason: "accounting-overflow" };
4710
+ }
4711
+ }
4712
+ retireOwnerGenerations(state);
4713
+ return { ok: true, changed: true };
4053
4714
  }
4054
4715
 
4055
4716
  function migrateV1GovernorState(raw, nowMs) {
@@ -4058,7 +4719,7 @@ function migrateV1GovernorState(raw, nowMs) {
4058
4719
  "intents", "reservations", "manualProbe",
4059
4720
  ]) || raw.version !== 1 || !isRecord(raw.budgets) || !isRecord(raw.reservations)) return null;
4060
4721
  const migrated = structuredClone(raw);
4061
- migrated.version = GOVERNOR_STATE_VERSION;
4722
+ migrated.version = 6;
4062
4723
  migrated.observers = { core: { etag: null, outcome: "idle", at: 0, nextAt: 0 } };
4063
4724
  // The v1 core value came from /rate_limit, which is not authoritative. Do not
4064
4725
  // preserve it as bootstrap evidence. GraphQL remains valid.
@@ -4086,7 +4747,8 @@ function migrateV1GovernorState(raw, nowMs) {
4086
4747
  ]));
4087
4748
  migrated.probeClaims = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, null]));
4088
4749
  migrated.fairness = { manualStreak: 0 };
4089
- return normalizeGovernorState(migrated, nowMs);
4750
+ const legacy = normalizeGovernorState(migrated, nowMs, { prune: false, acceptVersion: 6 });
4751
+ return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
4090
4752
  }
4091
4753
 
4092
4754
  function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
@@ -4099,11 +4761,21 @@ function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
4099
4761
  : { ok: false, reason: "unwritable" };
4100
4762
  }
4101
4763
  try {
4764
+ if (Buffer.byteLength(raw) > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "capacity" };
4102
4765
  const parsed = JSON.parse(raw);
4103
4766
  const normalized = normalizeGovernorState(parsed, nowMs);
4104
4767
  if (normalized) return { ok: true, value: normalized };
4105
4768
  const migrated = migrateGovernorState(parsed, nowMs);
4106
4769
  if (!migrated) return { ok: false, reason: "corrupt" };
4770
+ if (persistMigration && parsed.version === 6) {
4771
+ const backupPath = `${path}.pre-v7.backup`;
4772
+ try {
4773
+ writeFileSync(backupPath, raw, { encoding: "utf8", mode: 0o600, flag: "wx" });
4774
+ chmodSync(backupPath, 0o600);
4775
+ } catch (error) {
4776
+ if (error?.code !== "EEXIST") return { ok: false, reason: "unwritable" };
4777
+ }
4778
+ }
4107
4779
  if (!persistMigration) return { ok: true, value: migrated, migrated: true };
4108
4780
  // Read-only callers persist the exact migration while they still hold the
4109
4781
  // scope lock. Mutating callers use this same value for their final write.
@@ -4118,13 +4790,25 @@ function writeGovernorState(path, state) {
4118
4790
  const parent = dirname(path);
4119
4791
  let tempPath = null;
4120
4792
  try {
4793
+ if (state?.version === GOVERNOR_STATE_VERSION) {
4794
+ if (!Number.isSafeInteger(state.revision) || state.revision === Number.MAX_SAFE_INTEGER) {
4795
+ return { ok: false, reason: "accounting-overflow" };
4796
+ }
4797
+ state.revision += 1;
4798
+ }
4799
+ const serialized = serializeGovernorState(state);
4800
+ if (state?.version === GOVERNOR_STATE_VERSION &&
4801
+ Buffer.byteLength(serialized) > GOVERNOR_MAX_LEDGER_BYTES) {
4802
+ return { ok: false, reason: "capacity" };
4803
+ }
4121
4804
  mkdirSync(parent, { recursive: true, mode: 0o700 });
4122
4805
  chmodSync(parent, 0o700);
4123
4806
  tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
4124
- writeFileSync(tempPath, serializeGovernorState(state), { encoding: "utf8", mode: 0o600, flag: "wx" });
4807
+ writeFileSync(tempPath, serialized, { encoding: "utf8", mode: 0o600, flag: "wx" });
4125
4808
  chmodSync(tempPath, 0o600);
4809
+ // The private temporary inode already has mode 0600. Rename is the commit
4810
+ // point: no fallible operation may report failure after state replacement.
4126
4811
  renameSync(tempPath, path);
4127
- chmodSync(path, 0o600);
4128
4812
  return { ok: true };
4129
4813
  } catch {
4130
4814
  if (tempPath !== null) {
@@ -4134,6 +4818,15 @@ function writeGovernorState(path, state) {
4134
4818
  }
4135
4819
  }
4136
4820
 
4821
+ function writeGovernorQuotaState(path, state, nowMs) {
4822
+ const normalized = normalizeGovernorState(state, nowMs, { prune: false });
4823
+ if (!normalized) return { ok: false, reason: "corrupt" };
4824
+ if (!governorAccountingSafe(normalized, nowMs)) {
4825
+ return { ok: false, reason: "accounting-overflow" };
4826
+ }
4827
+ return writeGovernorState(path, normalized);
4828
+ }
4829
+
4137
4830
  function lockOwner(path) {
4138
4831
  try {
4139
4832
  const owner = JSON.parse(readFileSync(path, "utf8"));
@@ -4443,9 +5136,18 @@ function mutateGovernor(scope, nowMs, mutate) {
4443
5136
  const loaded = readGovernorState(scope.path, at, { persistMigration: false });
4444
5137
  if (!loaded.ok) return loaded;
4445
5138
  const state = loaded.value;
5139
+ if (!governorAccountingSafe(state, at)) return { ok: false, reason: "accounting-overflow" };
5140
+ const oldBootReconciled = reconcileBootDebt(state, at);
5141
+ if (!oldBootReconciled.ok) return oldBootReconciled;
5142
+ const compacted = compactGovernorReceipts(state, at);
5143
+ if (!compacted.ok) return compacted;
5144
+ const bootReconciled = reconcileBootDebt(state, at);
5145
+ if (!bootReconciled.ok) return bootReconciled;
4446
5146
  // A mutator can reject without writing. Keep an untouched migration so
4447
5147
  // that even that path upgrades the file once before releasing the lock.
4448
- const migration = loaded.migrated ? structuredClone(state) : null;
5148
+ const migration = loaded.migrated || compacted.changed ||
5149
+ oldBootReconciled.changed || bootReconciled.changed
5150
+ ? structuredClone(state) : null;
4449
5151
  const returnAfterMigration = (result) => {
4450
5152
  if (migration === null) return result;
4451
5153
  const written = writeGovernorState(scope.path, migration);
@@ -4453,6 +5155,7 @@ function mutateGovernor(scope, nowMs, mutate) {
4453
5155
  };
4454
5156
  const value = mutate(state, at);
4455
5157
  if (value?.ok === false && value.write !== true) return returnAfterMigration(value);
5158
+ if (!governorAccountingSafe(state, at)) return returnAfterMigration({ ok: false, reason: "accounting-overflow" });
4456
5159
  const normalized = normalizeGovernorState(state, at);
4457
5160
  if (!normalized) return returnAfterMigration({ ok: false, reason: "corrupt" });
4458
5161
  const written = writeGovernorState(scope.path, normalized);
@@ -4481,7 +5184,7 @@ const GOVERNOR_MAX_ATOMIC_COST = Math.max(...Object.values(OPERATION_COSTS)
4481
5184
  function returnPacingCredit(state, resource, unusedCost, nowMs) {
4482
5185
  const budget = state.budgets[resource];
4483
5186
  if (!budget || !(unusedCost > 0)) return;
4484
- const decision = resourceDecision({ budget, resource, nowMs, cost: 0, chargedCost: 0 });
5187
+ const decision = governorResourceDecision(state, resource, nowMs, 0);
4485
5188
  const callsPerMs = decision?.callsPerMs;
4486
5189
  if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) return;
4487
5190
  const credit = Math.min(unusedCost, GOVERNOR_MAX_ATOMIC_COST) / callsPerMs;
@@ -4544,6 +5247,10 @@ function scheduleGovernorState(state, nowMs) {
4544
5247
  leases: state.leases,
4545
5248
  budgets: state.budgets,
4546
5249
  reservations: Object.entries(state.reservations).map(([id, reservation]) => ({ id, ...reservation })),
5250
+ debtCosts: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
5251
+ state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
5252
+ reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs),
5253
+ ])),
4547
5254
  lanes,
4548
5255
  cursors,
4549
5256
  nowMs,
@@ -4572,6 +5279,7 @@ function scheduleGovernorState(state, nowMs) {
4572
5279
  startedAt: null,
4573
5280
  completedAt: null,
4574
5281
  outcome: null,
5282
+ receipt: null,
4575
5283
  };
4576
5284
  reservationCount += 1;
4577
5285
  delete state.intents[grant.intentId];
@@ -4676,6 +5384,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4676
5384
  dueAtCoreReset,
4677
5385
  );
4678
5386
  if (nextAt > at) return { value: { status: "waiting", nextAt } };
5387
+ if (resource === "graphql" && state.controlReceipts.graphql) {
5388
+ // The old claim's lease has expired. Keep an issued child charged as
5389
+ // debt, and fence the old capability so a new observer can make progress.
5390
+ if (!retireControlReceiptToDebt(state, "graphql", at)) {
5391
+ return { ok: false, reason: "accounting-overflow" };
5392
+ }
5393
+ }
4679
5394
  const nonce = randomUUID();
4680
5395
  const startedReservationIds = Object.entries(state.reservations)
4681
5396
  // An expired owner can no longer settle its request. Keep that uncertain
@@ -4686,6 +5401,36 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4686
5401
  state.leases[reservation.leaseId]?.expiresAt > at &&
4687
5402
  reservationCost(reservation, resource, state.leases, at) > 0)
4688
5403
  .map(([id]) => id);
5404
+ const quiescentGroups = Object.values(state.debtGroups).filter((group) =>
5405
+ group.quiescent && group.barrierAt < at &&
5406
+ (group.units[resource] > 0 || group.counts[resource] > 0));
5407
+ const barrierAt = Math.max(0, ...quiescentGroups.map((group) => group.barrierAt));
5408
+ const debtUnits = quiescentGroups.reduce((sum, group) => sum + group.units[resource], 0);
5409
+ const debtCount = quiescentGroups.reduce((sum, group) => sum + group.counts[resource], 0);
5410
+ if (!Number.isSafeInteger(debtUnits) || !Number.isSafeInteger(debtCount)) {
5411
+ return { ok: false, reason: "accounting-overflow" };
5412
+ }
5413
+ const debtSnapshot = {
5414
+ units: debtUnits,
5415
+ count: debtCount,
5416
+ barrierAt,
5417
+ revision: state.debt[resource].revision,
5418
+ };
5419
+ if (resource === "graphql") {
5420
+ state.controlReceipts.graphql = {
5421
+ leaseId, intentId: nonce, costs: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS },
5422
+ actualCosts: null, accountedCosts: { core: 0, graphql: 0 },
5423
+ notBefore: at, status: "started",
5424
+ epochs: { core: null, graphql: state.epochs.graphql },
5425
+ startedAt: at, completedAt: null, outcome: null,
5426
+ receipt: {
5427
+ scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
5428
+ ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
5429
+ deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
5430
+ allowance: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS }, dispatches: [],
5431
+ },
5432
+ };
5433
+ }
4689
5434
  state.probeClaims[resource] = {
4690
5435
  ownerLeaseId: leaseId,
4691
5436
  nonce,
@@ -4693,6 +5438,7 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4693
5438
  nextAt: at,
4694
5439
  claimAt: at,
4695
5440
  startedReservationIds,
5441
+ debtSnapshot,
4696
5442
  };
4697
5443
  state.observers[resource].outcome = "waiting";
4698
5444
  state.observers[resource].at = at;
@@ -4704,6 +5450,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
4704
5450
  leaseUntil: state.probeClaims[resource].leaseUntil,
4705
5451
  startedReservationIds,
4706
5452
  coreEtag: state.observers[resource].etag,
5453
+ ...(resource === "graphql" ? { receiptCapability: {
5454
+ reservationId: "control:graphql", controlResource: "graphql", leaseId,
5455
+ scopeHash: scope.hash,
5456
+ ownerNonce: state.controlReceipts.graphql.receipt.ownerNonce,
5457
+ generation: state.controlReceipts.graphql.receipt.generation,
5458
+ deadline: state.controlReceipts.graphql.receipt.deadline,
5459
+ } } : {}),
4707
5460
  } };
4708
5461
  });
4709
5462
  }
@@ -4785,6 +5538,34 @@ function budgetFromObservation(raw, previous, nowMs, {
4785
5538
  return { status: "accepted", budget: next, epochChanged };
4786
5539
  }
4787
5540
 
5541
+ function retireClaimedQuiescentDebt(state, resource, claim) {
5542
+ const snapshot = claim.debtSnapshot;
5543
+ if (snapshot.units === 0 && snapshot.count === 0) return true;
5544
+ if (claim.claimAt <= snapshot.barrierAt) return true;
5545
+ const eligible = Object.entries(state.debtGroups).filter(([, group]) =>
5546
+ group.quiescent && group.barrierAt < claim.claimAt &&
5547
+ group.barrierRevisions[resource] <= snapshot.revision &&
5548
+ (group.units[resource] > 0 || group.counts[resource] > 0));
5549
+ const units = eligible.reduce((total, [, group]) => total + group.units[resource], 0);
5550
+ const count = eligible.reduce((total, [, group]) => total + group.counts[resource], 0);
5551
+ const debt = state.debt[resource];
5552
+ if (!Number.isSafeInteger(units) || !Number.isSafeInteger(count) ||
5553
+ units !== snapshot.units || count !== snapshot.count ||
5554
+ debt.quiescentUnits < units || debt.quiescentCount < count ||
5555
+ !Number.isSafeInteger(debt.revision + 1)) return false;
5556
+ debt.quiescentUnits -= units;
5557
+ debt.quiescentCount -= count;
5558
+ debt.revision += 1;
5559
+ for (const [id, group] of eligible) {
5560
+ group.units[resource] = 0;
5561
+ group.counts[resource] = 0;
5562
+ if (RATE_RESOURCES.every((name) => group.units[name] === 0 && group.counts[name] === 0)) {
5563
+ delete state.debtGroups[id];
5564
+ }
5565
+ }
5566
+ return true;
5567
+ }
5568
+
4788
5569
  function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
4789
5570
  return mutateGovernor(scope, nowMs, (state, at) => {
4790
5571
  const claim = state.probeClaims[resource];
@@ -4833,6 +5614,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
4833
5614
  }
4834
5615
  }
4835
5616
  if (changedResources.length === 0) return { ok: false, reason: "corrupt" };
5617
+ if (!retireClaimedQuiescentDebt(state, resource, claim)) {
5618
+ return { ok: false, reason: "accounting-overflow" };
5619
+ }
4836
5620
  const completedBeforeClaim = Object.entries(state.reservations)
4837
5621
  .filter(([, reservation]) =>
4838
5622
  reservation.status === "completed" && reservation.completedAt < claim.claimAt);
@@ -4884,6 +5668,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
4884
5668
  state.budgets = nextBudgets;
4885
5669
  state.epochs = nextEpochs;
4886
5670
  state.probeClaims[resource] = null;
5671
+ if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
5672
+ state.controlReceipts.graphql = null;
5673
+ }
4887
5674
  {
4888
5675
  const published = nextBudgets[resource];
4889
5676
  state.observers[resource] = {
@@ -4944,6 +5731,15 @@ function failProbeClaim(scope, leaseId, nonce, nowMs, resource) {
4944
5731
  // also marked core failed would erase authority core had legitimately
4945
5732
  // established, and stall a lane with nothing wrong with it.
4946
5733
  state.probeClaims[resource] = null;
5734
+ if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
5735
+ const control = state.controlReceipts.graphql;
5736
+ if (control.receipt.dispatches.length === 0) state.controlReceipts.graphql = null;
5737
+ else if (control.receipt.dispatches.every((dispatch) => dispatch.terminalAt !== null)) {
5738
+ if (!retireControlReceiptToDebt(state, "graphql", at)) {
5739
+ return { ok: false, reason: "accounting-overflow" };
5740
+ }
5741
+ }
5742
+ }
4947
5743
  state.observers[resource] = {
4948
5744
  ...state.observers[resource],
4949
5745
  outcome: "failed",
@@ -5118,7 +5914,8 @@ function startReservation(scope, reservationId, nowMs) {
5118
5914
  }
5119
5915
  const charged = Object.values(state.reservations).reduce(
5120
5916
  (total, item) => total + reservationCost(item, resource, state.leases, at),
5121
- 0,
5917
+ state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
5918
+ reservationCost(state.controlReceipts[resource], resource, state.leases, at),
5122
5919
  );
5123
5920
  if (budget.remaining - resourceReserve(budget.limit) - charged < 0) {
5124
5921
  reservation.status = "cancelled";
@@ -5127,43 +5924,204 @@ function startReservation(scope, reservationId, nowMs) {
5127
5924
  }
5128
5925
  reservation.status = "started";
5129
5926
  reservation.startedAt = at;
5130
- return { value: { status: "started", reservationId } };
5927
+ const generation = state.ownerGenerations[reservation.leaseId] ?? governorId();
5928
+ state.ownerGenerations[reservation.leaseId] = generation;
5929
+ reservation.receipt = {
5930
+ scopeHash: scope.hash,
5931
+ ownerNonce: generation,
5932
+ generation,
5933
+ ownerPid: process.pid,
5934
+ ownerBirth: coordinationProcessBirth(),
5935
+ bootId: coordinationBootId(),
5936
+ deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
5937
+ allowance: { ...reservation.costs },
5938
+ dispatches: [],
5939
+ };
5940
+ return { value: { status: "started", reservationId,
5941
+ receiptCapability: receiptCapabilityFromReservation(reservationId, reservation) } };
5131
5942
  });
5132
5943
  }
5133
5944
 
5134
- function completeReservation(scope, reservationId, completion, nowMs) {
5945
+ function mutateReceiptDispatch(scope, capability, nowMs, mutate) {
5946
+ if (!capability || capability.scopeHash !== scope?.hash ||
5947
+ !validGovernorId(capability.ownerNonce) || !validGovernorId(capability.generation) ||
5948
+ !validGovernorId(capability.leaseId) ||
5949
+ typeof capability.reservationId !== "string") return { ok: false, reason: "receipt-invalid" };
5135
5950
  return mutateGovernor(scope, nowMs, (state, at) => {
5136
- const reservation = state.reservations[reservationId];
5137
- if (!reservation || reservation.status !== "started" || !GOVERNOR_OUTCOMES.has(completion?.outcome)) {
5138
- return { ok: false, reason: "stale" };
5139
- }
5140
- const suppliedActual = completion.actualCost === undefined
5141
- ? null : exactResourceCosts(completion.actualCost);
5142
- if (completion.actualCost !== undefined && !suppliedActual) return { ok: false, reason: "corrupt" };
5143
- const measured = completion.outcome === "measured-success"
5144
- ? suppliedActual
5145
- : reservation.costs;
5146
- if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
5147
- return { ok: false, reason: "corrupt" };
5148
- }
5149
- reservation.status = "completed";
5150
- reservation.completedAt = at;
5151
- reservation.outcome = completion.outcome;
5152
- reservation.actualCosts = { ...measured };
5153
- reservation.accountedCosts = { core: 0, graphql: 0 };
5154
- // The same rule as the observing settlement path: this one narrows the
5155
- // charge in exactly the same way, so it releases the pacing in exactly the
5156
- // same way. Wiring it into only one of the two settlement paths left every
5157
- // doctor probe and manual operation paying for capacity it did not use.
5158
- if (completion.outcome === "measured-success") {
5159
- for (const resource of RATE_RESOURCES) {
5160
- returnPacingCredit(state, resource, reservation.costs[resource] - measured[resource], at);
5951
+ const controlResource = capability.controlResource;
5952
+ if (controlResource !== undefined && !RATE_RESOURCES.includes(controlResource)) {
5953
+ return { ok: false, reason: "receipt-invalid" };
5954
+ }
5955
+ const reservation = controlResource
5956
+ ? state.controlReceipts[controlResource] : state.reservations[capability.reservationId];
5957
+ const receipt = reservation?.receipt;
5958
+ if (!receipt || receipt.scopeHash !== capability.scopeHash ||
5959
+ receipt.ownerNonce !== capability.ownerNonce || receipt.generation !== capability.generation ||
5960
+ reservation.leaseId !== capability.leaseId || receipt.deadline !== capability.deadline) {
5961
+ return { ok: false, reason: "compacted" };
5962
+ }
5963
+ return mutate(state, reservation, receipt, at);
5964
+ });
5965
+ }
5966
+
5967
+ function issueGovernorDispatch(scope, capability, operation, nowMs) {
5968
+ const costs = operationCost(operation);
5969
+ if (!costs) return { ok: false, reason: "undeclared" };
5970
+ if ([...terminalizationBacklog.keys()].filter((key) =>
5971
+ key.startsWith(`${scope?.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
5972
+ return { ok: false, reason: "completion-capacity" };
5973
+ }
5974
+ return mutateReceiptDispatch(scope, capability, nowMs, (state, reservation, receipt, at) => {
5975
+ if (reservation.status !== "started" || receipt.deadline <= at ||
5976
+ (!capability.controlResource && (
5977
+ state.ownerGenerations[reservation.leaseId] !== receipt.generation ||
5978
+ state.leases[reservation.leaseId]?.expiresAt <= at ||
5979
+ !state.leases[reservation.leaseId]))) return { ok: false, reason: "receipt-expired" };
5980
+ const issued = { core: 0, graphql: 0 };
5981
+ for (const dispatch of receipt.dispatches) {
5982
+ for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
5983
+ }
5984
+ if (RATE_RESOURCES.some((resource) =>
5985
+ !Number.isSafeInteger(issued[resource] + costs[resource]) ||
5986
+ issued[resource] + costs[resource] > receipt.allowance[resource])) {
5987
+ return { ok: false, reason: "receipt-allowance" };
5988
+ }
5989
+ const sequence = receipt.dispatches.length + 1;
5990
+ if (sequence > 32) return { ok: false, reason: "receipt-allowance" };
5991
+ receipt.dispatches.push({ sequence, operation, costs: { ...costs },
5992
+ issuedAt: at, terminalAt: null, childPid: null, childBirth: null, neverStarted: false });
5993
+ return { value: { sequence, deadline: receipt.deadline } };
5994
+ });
5995
+ }
5996
+
5997
+ function markGovernorDispatchChild(scope, capability, sequence, pid, nowMs) {
5998
+ if (!Number.isSafeInteger(pid) || pid < 1) return { ok: false, reason: "child-unknown" };
5999
+ return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
6000
+ (_state, _reservation, receipt) => {
6001
+ const dispatch = receipt.dispatches[sequence - 1];
6002
+ if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
6003
+ dispatch.childPid = pid;
6004
+ dispatch.childBirth = coordinationProcessBirth(pid);
6005
+ return { value: { status: "recorded" } };
6006
+ });
6007
+ }
6008
+
6009
+ function terminalGovernorDispatch(scope, capability, sequence, nowMs, { neverStarted = false } = {}) {
6010
+ return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
6011
+ (_state, _reservation, receipt, at) => {
6012
+ const dispatch = receipt.dispatches[sequence - 1];
6013
+ if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
6014
+ if (neverStarted && dispatch.childPid !== null) return { ok: false, reason: "child-started" };
6015
+ if (dispatch.terminalAt === null) {
6016
+ dispatch.terminalAt = at;
6017
+ dispatch.neverStarted = neverStarted;
5161
6018
  }
5162
- }
5163
- return { value: { status: "completed", actualCosts: reservation.actualCosts } };
6019
+ return { value: { status: "terminal" } };
6020
+ });
6021
+ }
6022
+
6023
+ const activeOwnerDispatches = new Map();
6024
+ function ownerDispatchKey(scope, capability) {
6025
+ return `${scope.hash}:${capability.generation}`;
6026
+ }
6027
+
6028
+ function acknowledgeSealedGeneration(scope, capability, sealedNonce, nowMs) {
6029
+ const key = ownerDispatchKey(scope, capability);
6030
+ if ((activeOwnerDispatches.get(key) ?? 0) !== 0) return { ok: false, reason: "children-active" };
6031
+ return mutateGovernor({ ...scope, identityProvider: null }, nowMs, (state, at) => {
6032
+ const group = state.debtGroups[capability.generation];
6033
+ if (!group) return { value: { status: "absent" } };
6034
+ if (group.nonce !== sealedNonce) return { ok: false, reason: "stale" };
6035
+ if (group.quiescent) return { value: { status: "acknowledged", nonce: group.nonce } };
6036
+ if (!group.ownerNonce || group.ownerNonce !== capability.ownerNonce ||
6037
+ !group.ownerBirth || !group.bootId ||
6038
+ group.ownerPid !== process.pid || group.ownerBirth !== coordinationProcessBirth() ||
6039
+ group.bootId !== coordinationBootId() ||
6040
+ Object.values(state.reservations).some((reservation) =>
6041
+ reservation.receipt?.generation === capability.generation)) {
6042
+ return { ok: false, reason: "quiescence-unverified" };
6043
+ }
6044
+ if (!makeDebtGroupQuiescent(state, group, at)) return { ok: false, reason: "accounting-overflow" };
6045
+ return { value: { status: "acknowledged", nonce: group.nonce } };
5164
6046
  });
5165
6047
  }
5166
6048
 
6049
+ function completeReservation(scope, reservationId, completion, nowMs) {
6050
+ const reservation = inspectGovernor({ ...scope, identityProvider: null }, nowMs);
6051
+ if (!reservation.ok) return reservation;
6052
+ const leaseId = reservation.value.reservations[reservationId]?.leaseId;
6053
+ if (!leaseId) return { ok: false, reason: "stale" };
6054
+ return settleReservationWithBudgetObservations(scope, leaseId, reservationId, {
6055
+ ...completion,
6056
+ ...(completion.actualCost ? { actualCosts: completion.actualCost } : {}),
6057
+ observations: [],
6058
+ }, nowMs);
6059
+ }
6060
+
6061
+ function terminalizeReservation(scope, capability, completion, nowMs) {
6062
+ if (!capability?.reservationId || !capability.leaseId) {
6063
+ return { ok: false, reason: "blocked", status: "blocked" };
6064
+ }
6065
+ const result = settleReservationWithBudgetObservations(
6066
+ { ...scope, identityProvider: null }, capability.leaseId,
6067
+ capability.reservationId, { ...completion, capability }, nowMs,
6068
+ );
6069
+ if (result.ok) return { ok: true, value: { ...result.value, status: "settled" } };
6070
+ const status = ["busy", "unwritable"].includes(result.reason) ? "retryable"
6071
+ : result.reason === "stale" ? "compacted" : "blocked";
6072
+ return { ...result, status };
6073
+ }
6074
+
6075
+ const terminalizationBacklog = new Map();
6076
+ function terminalizationBacklogForResource(backlog, scopeHash, resource) {
6077
+ if (!scopeHash) return 0;
6078
+ let count = 0;
6079
+ for (const item of backlog.values()) {
6080
+ if (item.scope.hash !== scopeHash) continue;
6081
+ if (item.capability?.costs?.[resource] > 0 ||
6082
+ !RATE_RESOURCES.some((name) => item.capability?.costs?.[name] > 0)) count += 1;
6083
+ }
6084
+ return count;
6085
+ }
6086
+ function receiptCapabilityFromReservation(reservationId, reservation) {
6087
+ if (!reservation?.receipt || !reservation?.costs) return null;
6088
+ return {
6089
+ reservationId, leaseId: reservation.leaseId,
6090
+ scopeHash: reservation.receipt.scopeHash,
6091
+ ownerNonce: reservation.receipt.ownerNonce,
6092
+ generation: reservation.receipt.generation,
6093
+ deadline: reservation.receipt.deadline,
6094
+ costs: { ...reservation.costs },
6095
+ };
6096
+ }
6097
+ function queueTerminalization(scope, capability, completion, now = Date.now) {
6098
+ const frozen = { ...scope, identityProvider: null };
6099
+ const key = `${scope.hash}:${capability?.reservationId ?? "invalid"}`;
6100
+ const first = terminalizeReservation(frozen, capability, completion, now());
6101
+ if (first.ok || first.status !== "retryable") return first;
6102
+ if (terminalizationBacklog.has(key)) return first;
6103
+ if ([...terminalizationBacklog.keys()].filter((existing) =>
6104
+ existing.startsWith(`${scope.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
6105
+ return { ok: false, reason: "completion-capacity", status: "blocked" };
6106
+ }
6107
+ const item = { scope: frozen, capability, completion, now, attempt: 0, timer: null };
6108
+ const retry = () => {
6109
+ const result = terminalizeReservation(item.scope, item.capability, item.completion, item.now());
6110
+ if (result.ok || result.status !== "retryable") {
6111
+ terminalizationBacklog.delete(key);
6112
+ return;
6113
+ }
6114
+ const delays = [1000, 2000, 4000, 5000];
6115
+ const delay = delays[Math.min(item.attempt++, delays.length - 1)];
6116
+ item.timer = setTimeout(retry, delay + Math.floor(Math.random() * 100));
6117
+ item.timer.unref?.();
6118
+ };
6119
+ terminalizationBacklog.set(key, item);
6120
+ item.timer = setTimeout(retry, 1000);
6121
+ item.timer.unref?.();
6122
+ return first;
6123
+ }
6124
+
5167
6125
  function settleReservationWithBudgetObservations(
5168
6126
  scope,
5169
6127
  leaseId,
@@ -5173,13 +6131,33 @@ function settleReservationWithBudgetObservations(
5173
6131
  ) {
5174
6132
  return mutateGovernor(scope, nowMs, (state, at) => {
5175
6133
  const reservation = state.reservations[reservationId];
5176
- const lease = state.leases[leaseId];
6134
+ if (reservation?.status === "completed" && reservation.leaseId === leaseId &&
6135
+ (!completion?.capability ||
6136
+ completion.capability.ownerNonce === reservation.receipt?.ownerNonce &&
6137
+ completion.capability.generation === reservation.receipt?.generation)) {
6138
+ return { value: { status: "completed", actualCosts: reservation.actualCosts,
6139
+ accountedCosts: reservation.accountedCosts } };
6140
+ }
5177
6141
  if (
5178
- !lease || lease.expiresAt <= at || !reservation ||
6142
+ !reservation ||
5179
6143
  reservation.leaseId !== leaseId || reservation.status !== "started" ||
5180
6144
  !GOVERNOR_OUTCOMES.has(completion?.outcome)
5181
6145
  ) return { ok: false, reason: "stale" };
5182
- const measured = completion.outcome === "measured-success"
6146
+ if (completion.capability && (completion.capability.scopeHash !== scope.hash ||
6147
+ completion.capability.reservationId !== reservationId ||
6148
+ completion.capability.leaseId !== leaseId ||
6149
+ completion.capability.ownerNonce !== reservation.receipt?.ownerNonce ||
6150
+ completion.capability.generation !== reservation.receipt?.generation ||
6151
+ completion.capability.deadline !== reservation.receipt?.deadline)) {
6152
+ return { ok: false, reason: "receipt-invalid" };
6153
+ }
6154
+ const neverIssued = completion.capability && (
6155
+ completion.neverStarted === true && reservation.receipt.dispatches.length === 0 ||
6156
+ reservation.receipt.dispatches.length > 0 &&
6157
+ reservation.receipt.dispatches.every((dispatch) => dispatch.neverStarted));
6158
+ const outcome = neverIssued ? "measured-success" : completion.outcome;
6159
+ const measured = neverIssued ? { core: 0, graphql: 0 }
6160
+ : outcome === "measured-success"
5183
6161
  ? exactResourceCosts(completion.actualCosts ?? completion.actualCost)
5184
6162
  : reservation.costs;
5185
6163
  if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
@@ -5231,17 +6209,17 @@ function settleReservationWithBudgetObservations(
5231
6209
  }
5232
6210
  reservation.status = "completed";
5233
6211
  reservation.completedAt = at;
5234
- reservation.outcome = completion.outcome;
6212
+ reservation.outcome = outcome;
5235
6213
  reservation.actualCosts = { ...measured };
5236
6214
  reservation.accountedCosts = Object.fromEntries(RATE_RESOURCES.map((resource) => [
5237
6215
  resource,
5238
- completion.outcome === "measured-success"
6216
+ outcome === "measured-success"
5239
6217
  ? Math.min(measured[resource], acceptedCosts[resource])
5240
6218
  : 0,
5241
6219
  ]));
5242
6220
  for (const resource of RATE_RESOURCES) {
5243
6221
  const budget = state.budgets[resource];
5244
- if (!budget || completion.outcome !== "measured-success") continue;
6222
+ if (!budget || outcome !== "measured-success") continue;
5245
6223
  // accountedCosts removes the portion already present in an authoritative
5246
6224
  // counter from the reservation residual. knownLocalUsed contains only
5247
6225
  // that same portion when it arrived after the factor baseline. Their sum
@@ -5255,7 +6233,7 @@ function settleReservationWithBudgetObservations(
5255
6233
  // loss proves nothing, so its worst case stays charged and its pacing stays
5256
6234
  // spent -- refunding there would let a run of timeouts pace as though
5257
6235
  // nothing had been sent.
5258
- if (completion.outcome === "measured-success") {
6236
+ if (outcome === "measured-success") {
5259
6237
  for (const resource of RATE_RESOURCES) {
5260
6238
  returnPacingCredit(state, resource, reservation.costs[resource] - measured[resource], at);
5261
6239
  }
@@ -5341,6 +6319,40 @@ function governorHealth(result, nowMs = Date.now()) {
5341
6319
  };
5342
6320
  }
5343
6321
 
6322
+ // Plain doctor must not call the identity coordinator: even its inspection
6323
+ // path may retire old registry attempts and persist the resulting registry.
6324
+ // A cached identity is useful only when it identifies one quota account for
6325
+ // the requested host. Multiple accounts cannot be resolved without probing.
6326
+ function doctorCachedQuotaScope(host, { root = identityRegistryRoot(), nowMs = Date.now() } = {}) {
6327
+ const registryPath = join(root, "registry.json");
6328
+ let registry;
6329
+ try {
6330
+ const file = lstatSync(registryPath);
6331
+ if (!file.isFile() || file.size > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "unsafe registry" };
6332
+ const parsed = JSON.parse(readFileSync(registryPath, "utf8"));
6333
+ registry = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
6334
+ } catch (error) {
6335
+ return { ok: false, reason: error?.code === "ENOENT" ? "cached identity missing" : "cached registry unreadable" };
6336
+ }
6337
+ if (!registry) return { ok: false, reason: "cached registry corrupt" };
6338
+ const quotas = new Set(Object.values(registry.identities)
6339
+ .filter((identity) => identity.host === host)
6340
+ .map((identity) => identity.quotaKey));
6341
+ if (quotas.size === 0) return { ok: false, reason: "cached identity missing" };
6342
+ if (quotas.size !== 1) return { ok: false, reason: "ambiguous cached identities" };
6343
+ const quotaKey = quotas.values().next().value;
6344
+ const path = join(root, `quota-${quotaKey}.json`);
6345
+ try {
6346
+ if (!lstatSync(path).isFile()) return { ok: false, reason: "unsafe quota path", path };
6347
+ } catch (error) {
6348
+ return { ok: false, reason: error?.code === "ENOENT" ? "cached quota missing" : "cached quota unreadable", path };
6349
+ }
6350
+ const result = readGovernorState(path, nowMs, { persistMigration: false });
6351
+ return result.ok && !result.missing
6352
+ ? { ok: true, result, path, quotaKey }
6353
+ : { ok: false, reason: result.missing ? "cached quota missing" : result.reason, path };
6354
+ }
6355
+
5344
6356
  async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
5345
6357
  let result = run();
5346
6358
  while (!result.ok && result.reason === "busy" && !signal?.aborted && now() < deadline) {
@@ -5470,7 +6482,8 @@ async function refreshResourceBudget(scope, leaseId, signal, resource, {
5470
6482
  let budgets;
5471
6483
  try {
5472
6484
  try { onObserverCall?.(resource); } catch { /* diagnostics cannot block a budget observer */ }
5473
- budgets = await requestIdentityStorage.run(scope, () => readBudgets(signal, scope.host, {
6485
+ budgets = await requestIdentityStorage.run({ ...scope,
6486
+ controlReceiptCapability: claim.value.receiptCapability }, () => readBudgets(signal, scope.host, {
5474
6487
  resources: [resource],
5475
6488
  coreEtag,
5476
6489
  renewClaim: async () => {
@@ -6074,7 +7087,7 @@ const SSH_RECONNECT_STEPS_MS = Object.freeze([1_000, 2_000, 4_000, 8_000, 16_000
6074
7087
  const SSH_STDERR_MAX_BYTES = 4_096;
6075
7088
  const SSH_ALIAS_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$/;
6076
7089
  const COLLECTOR_HOLD_REASONS = new Set([
6077
- "observer", "primary", "secondary", "disconnected", "coordination", "shared-wait", "cache-only",
7090
+ "observer", "primary", "secondary", "disconnected", "app-auth", "coordination", "shared-wait", "cache-only",
6078
7091
  ]);
6079
7092
  const COLLECTOR_ROW_KEYS = Object.freeze({
6080
7093
  actions: ["databaseId", "displayTitle", "workflowName", "number", "headBranch", "status", "conclusion",
@@ -6592,7 +7605,7 @@ function persistGitHubAppIdentity(root, identity, now = Date.now()) {
6592
7605
  const initialized = withGovernorLock(scope, () => {
6593
7606
  const loaded = readIdentityQuotaState(state, scope, now);
6594
7607
  if (!loaded.ok) return loaded;
6595
- return loaded.missing ? writeGovernorState(scope.path, loaded.value) : { ok: true };
7608
+ return loaded.missing ? writeGovernorQuotaState(scope.path, loaded.value, now) : { ok: true };
6596
7609
  });
6597
7610
  if (!initialized.ok) return initialized;
6598
7611
  state.identities[identity.credentialKey] = { host: identity.host, kind: identity.kind,
@@ -8403,7 +9416,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
8403
9416
  if (!["changed", "unchanged"].includes(transition.kind)) {
8404
9417
  const error = new Error("collector received unusable data");
8405
9418
  error.requestMetrics = acquisitionRequestMetrics(result);
8406
- error.uncertainReceipts = result.uncertainReceipts ?? [];
8407
9419
  throw error;
8408
9420
  }
8409
9421
  const rows = transition.kind === "changed" ? transition.data : previous?.rows;
@@ -8435,7 +9447,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
8435
9447
  hold: null,
8436
9448
  capabilities: result.capabilities ?? previous?.capabilities ?? {},
8437
9449
  requestMetrics: acquisitionRequestMetrics(result),
8438
- uncertainReceipts: result.uncertainReceipts ?? [],
8439
9450
  repositoryIdentity: result.repositoryIdentity,
8440
9451
  meta,
8441
9452
  securityNotes: resource === "security"
@@ -8616,6 +9627,7 @@ function createCollectorAcquisitionRuntime({
8616
9627
  },
8617
9628
  });
8618
9629
  const reservationId = admitted.reservationId;
9630
+ const transportScope = { ...context.scope, receiptCapability: admitted.receiptCapability };
8619
9631
  const governor = inspectGovernor(context.scope, now());
8620
9632
  const started = await markStarted({
8621
9633
  reservationId,
@@ -8623,8 +9635,9 @@ function createCollectorAcquisitionRuntime({
8623
9635
  epochs: governor.ok ? governor.value.epochs : {},
8624
9636
  });
8625
9637
  if (!started.ok || started.value?.status !== "started") {
8626
- completeReservation({ ...context.scope, identityProvider: null }, reservationId,
8627
- { outcome: "rejected" }, now());
9638
+ const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
9639
+ { outcome: "rejected", neverStarted: true }, now);
9640
+ if (terminal.status === "blocked") throw new Error("collector receipt settlement blocked");
8628
9641
  throw new Error("collector acquisition start failed");
8629
9642
  }
8630
9643
  const descriptor = tabForKey(resource);
@@ -8636,20 +9649,18 @@ function createCollectorAcquisitionRuntime({
8636
9649
  signal,
8637
9650
  AbortSignal.timeout(ACQUISITION_STARTED_DEADLINE_MS),
8638
9651
  ]);
8639
- let result = await requestIdentityStorage.run(context.scope, () => descriptor.fetch({
9652
+ let result = await requestIdentityStorage.run(transportScope, () => descriptor.fetch({
8640
9653
  signal: boundedSignal,
8641
9654
  entities,
8642
9655
  previousRaw: force ? null : snapshot?.raw ?? null,
8643
9656
  force,
8644
9657
  pages: demand.pages,
8645
- governor: { scope: context.scope, leaseId: context.leaseId },
9658
+ governor: { scope: transportScope, leaseId: context.leaseId },
8646
9659
  ...(securityPolicy ? { sources: securityPolicy.sources } : {}),
8647
9660
  }));
8648
9661
  if (securityPolicy) result = applyGitHubAppSecurityPolicy(result, securityPolicy);
8649
- settleReservationWithBudgetObservations(
8650
- { ...context.scope, identityProvider: null },
8651
- context.leaseId,
8652
- reservationId,
9662
+ const terminal = queueTerminalization(
9663
+ context.scope, admitted.receiptCapability,
8653
9664
  result?.measuredSuccess === false
8654
9665
  ? { outcome: "rejected", observations: result.observations ?? [] }
8655
9666
  : {
@@ -8660,8 +9671,11 @@ function createCollectorAcquisitionRuntime({
8660
9671
  },
8661
9672
  observations: result.observations ?? [],
8662
9673
  },
8663
- now(),
9674
+ now,
8664
9675
  );
9676
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
9677
+ throw new Error(`collector receipt settlement ${terminal.status}`);
9678
+ }
8665
9679
  const completedAt = now();
8666
9680
  const publication = collectorPublicationFromResult(
8667
9681
  resource, result, snapshot, demand, completedAt, claim.state,
@@ -8677,10 +9691,11 @@ function createCollectorAcquisitionRuntime({
8677
9691
  if (reconciliation !== null) publication.nextDueAt = completedAt + reconciliation;
8678
9692
  return publication;
8679
9693
  } catch (error) {
8680
- settleReservationWithBudgetObservations(
8681
- { ...context.scope, identityProvider: null }, context.leaseId, reservationId,
8682
- { outcome: governorOutcomeForError(error), observations: error.budgetObservations ?? [] }, now(),
8683
- );
9694
+ const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
9695
+ { outcome: governorOutcomeForError(error), observations: error.budgetObservations ?? [] }, now);
9696
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
9697
+ error.receiptSettlement = terminal.status;
9698
+ }
8684
9699
  throw error;
8685
9700
  }
8686
9701
  }
@@ -8794,6 +9809,9 @@ function createCollectorAcquisitionRuntime({
8794
9809
  item.nextDueAt = snapshot.nextDueAt;
8795
9810
  if (Number.isSafeInteger(snapshot.generation)) item.lastDeliveredGeneration = snapshot.generation;
8796
9811
  if (Number.isFinite(snapshot.lastSuccessAt)) item.lastDeliveredSuccessAt = snapshot.lastSuccessAt;
9812
+ if (publication && item.requestedAt !== null && snapshot.lastSuccessAt >= item.requestedAt) {
9813
+ item.requestedAt = null;
9814
+ }
8797
9815
  item.onSnapshot(snapshot);
8798
9816
  if (!publication || !Number.isSafeInteger(snapshot.generation)) return true;
8799
9817
  for (const waiter of item.invalidations) {
@@ -8809,11 +9827,17 @@ function createCollectorAcquisitionRuntime({
8809
9827
  }
8810
9828
 
8811
9829
  function startPoll(item, { manual = false, force = false } = {}) {
9830
+ if (closed || item.closed) return null;
9831
+ if (manual && !demandEnabled(item.demand)) item.requestedAt ??= now();
9832
+ if (!item.id) {
9833
+ if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
9834
+ return startInitialize(item);
9835
+ }
8812
9836
  if (item.pollPromise) {
8813
9837
  if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
8814
9838
  return item.pollPromise;
8815
9839
  }
8816
- if (closed || item.closed || !demandEnabled(item.demand)) return null;
9840
+ if (!demandEnabled(item.demand) && item.requestedAt === null) return null;
8817
9841
  item.pollPromise = poll(item, { manual, force }).finally(() => {
8818
9842
  item.pollPromise = null;
8819
9843
  if (item.promotionGeneration !== null) {
@@ -8832,7 +9856,7 @@ function createCollectorAcquisitionRuntime({
8832
9856
  }
8833
9857
 
8834
9858
  function schedule(item, delay, operation = () => startPoll(item)) {
8835
- if (closed || item.closed || !demandEnabled(item.demand)) return;
9859
+ if (closed || item.closed || (!demandEnabled(item.demand) && item.requestedAt === null)) return;
8836
9860
  if (item.timer !== null) clearTimeout_(item.timer);
8837
9861
  item.timer = setTimeout_(() => {
8838
9862
  item.timer = null;
@@ -8898,7 +9922,8 @@ function createCollectorAcquisitionRuntime({
8898
9922
  () => engine.currentClaim(subscriptionId, claim), () => producer({
8899
9923
  target: item.target,
8900
9924
  resource: item.resource,
8901
- demand: { ...ownership.value.demand, background: item.demand.background },
9925
+ demand: { ...ownership.value.demand, background: item.demand.background,
9926
+ active: ownership.value.demand?.active || item.requestedAt !== null },
8902
9927
  snapshot: ownership.value.snapshot,
8903
9928
  claim: { ...claim, state: item },
8904
9929
  provider: item.provider,
@@ -8919,12 +9944,11 @@ function createCollectorAcquisitionRuntime({
8919
9944
  engine.refresh(subscriptionId, { claim, failure: {
8920
9945
  hold: "disconnected",
8921
9946
  requestMetrics: produced.requestMetrics ?? {},
8922
- uncertainReceipts: produced.uncertainReceipts ?? [],
8923
9947
  } });
8924
9948
  engine.unsubscribe(subscriptionId);
8925
9949
  item.id = null;
8926
9950
  targetInitializations.delete(targetInitializationKey(item.target));
8927
- item.onHold?.("disconnected");
9951
+ item.onHold?.("app-auth");
8928
9952
  retryIn = 1;
8929
9953
  return;
8930
9954
  }
@@ -8937,7 +9961,6 @@ function createCollectorAcquisitionRuntime({
8937
9961
  engine.refresh(subscriptionId, { claim, failure: {
8938
9962
  hold: "disconnected",
8939
9963
  requestMetrics: produced.requestMetrics ?? {},
8940
- uncertainReceipts: produced.uncertainReceipts ?? [],
8941
9964
  } });
8942
9965
  item.onHold?.("disconnected");
8943
9966
  retryIn = item.demand.floorMs;
@@ -8960,10 +9983,12 @@ function createCollectorAcquisitionRuntime({
8960
9983
  else if (claim) engine.refresh(subscriptionId, { claim, failure: {
8961
9984
  hold,
8962
9985
  requestMetrics: error?.requestMetrics ?? {},
8963
- uncertainReceipts: error?.uncertainReceipts ?? [],
8964
9986
  } });
8965
9987
  else engine.setHold(subscriptionId, hold, { resource: item.resource, accessKey: item.identity.accessKey });
8966
- item.onHold?.(hold);
9988
+ item.onHold?.(item.provider?.definition?.type === "github-app" &&
9989
+ (error?.apiResponse?.status === 401 || error?.message === "GitHub App token unavailable" ||
9990
+ classify(error) === "auth-problem")
9991
+ ? "app-auth" : hold);
8967
9992
  try { onDiagnostic({ stage: "acquire", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
8968
9993
  retryIn = error?.notStarted && Number.isFinite(error.retryAt)
8969
9994
  ? Math.max(50, error.retryAt - now()) : Math.min(1_000, item.demand.floorMs);
@@ -8979,13 +10004,16 @@ function createCollectorAcquisitionRuntime({
8979
10004
  if (item.id && item.identity?.accessKey) {
8980
10005
  engine.setHold(item.id, hold, { resource: item.resource, accessKey: item.identity.accessKey });
8981
10006
  }
8982
- item.onHold?.(hold);
10007
+ item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
10008
+ error?.message === "GitHub App token unavailable" ? "app-auth" : hold);
8983
10009
  try { onDiagnostic({ stage: "provider-refresh", reason: redact(shortErr(error)) }); } catch { /* isolated */ }
8984
10010
  retryIn = Math.min(1_000, item.demand.floorMs);
8985
10011
  } finally {
8986
10012
  item.polling = false;
8987
10013
  item.pollPhase = null;
8988
- schedule(item, retryIn, item.id ? () => startPoll(item) : () => startInitialize(item));
10014
+ schedule(item, retryIn, item.id
10015
+ ? () => startPoll(item, { manual: item.requestedAt !== null })
10016
+ : () => startInitialize(item));
8989
10017
  }
8990
10018
  }
8991
10019
 
@@ -9018,12 +10046,16 @@ function createCollectorAcquisitionRuntime({
9018
10046
  if (subscription.value.snapshot && !accessReset) {
9019
10047
  deliverCollectorSnapshot(item, subscription.value.snapshot, bindingRevision);
9020
10048
  }
9021
- if (demandEnabled(item.demand)) {
9022
- await startPoll(item, { manual: item.invalidations.size > 0 });
10049
+ if (demandEnabled(item.demand) || item.requestedAt !== null) {
10050
+ const queued = item.queuedRefresh;
10051
+ item.queuedRefresh = null;
10052
+ await startPoll(item, { manual: item.invalidations.size > 0 || item.requestedAt !== null || Boolean(queued),
10053
+ force: queued?.force === true });
9023
10054
  }
9024
10055
  } catch (error) {
9025
10056
  try { onDiagnostic({ stage: "initialize", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
9026
- item.onHold?.("disconnected");
10057
+ item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
10058
+ error?.message === "GitHub App token unavailable" ? "app-auth" : "disconnected");
9027
10059
  schedule(item, 1_000, () => startInitialize(item));
9028
10060
  }
9029
10061
  }
@@ -9040,7 +10072,7 @@ function createCollectorAcquisitionRuntime({
9040
10072
  if (!normalizedDemand) throw new Error("invalid collector demand");
9041
10073
  const item = { target, resource, demand: normalizedDemand, onSnapshot, onHold, id: null, identity: null,
9042
10074
  provider: null, polling: false, pollPhase: null, pollPromise: null, controller: null,
9043
- initializePromise: null, closed: false, timer: null, queuedRefresh: null,
10075
+ initializePromise: null, closed: false, timer: null, queuedRefresh: null, requestedAt: null,
9044
10076
  bindingRevision: 0, invalidations: new Set(), lastDeliveredGeneration: null,
9045
10077
  lastDeliveredSuccessAt: null, promotionGeneration: null, promotionAt: null };
9046
10078
  items.add(item);
@@ -9349,7 +10381,14 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9349
10381
  },
9350
10382
  });
9351
10383
  const write = (message) => {
9352
- if (welcomed && socket?.writable) socket.write(encodeCollectorFrame(message));
10384
+ if (!welcomed || !socket?.writable) return false;
10385
+ socket.write(encodeCollectorFrame(message));
10386
+ return true;
10387
+ };
10388
+ const flushRefresh = (item) => {
10389
+ if (item.pendingRefresh !== null && write({ type: "refresh", id: item.id, force: item.pendingRefresh })) {
10390
+ item.pendingRefresh = null;
10391
+ }
9353
10392
  };
9354
10393
  const resubscribe = (resetGenerations) => {
9355
10394
  for (const item of subscriptions.values()) {
@@ -9359,6 +10398,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9359
10398
  }
9360
10399
  write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
9361
10400
  resource: item.resource, demand: item.demand });
10401
+ flushRefresh(item);
9362
10402
  }
9363
10403
  };
9364
10404
  const connect = () => {
@@ -9449,7 +10489,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9449
10489
  return { ok: false, reason: "invalid" };
9450
10490
  }
9451
10491
  const item = { ...subscription,
9452
- onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1,
10492
+ onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1, pendingRefresh: null,
9453
10493
  sourceAge: createCollectorSourceAgeTracker({ now, monotonicNow, checkpoint: sourceCheckpoint }) };
9454
10494
  subscriptions.set(id, item);
9455
10495
  if (welcomed) write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
@@ -9467,7 +10507,10 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
9467
10507
  write({ type: "demand", id, demand: normalized });
9468
10508
  return { ok: true, changed: true };
9469
10509
  },
9470
- refresh(force = false) { write({ type: "refresh", id, force: force === true }); },
10510
+ refresh(force = false) {
10511
+ item.pendingRefresh = force === true || item.pendingRefresh === true;
10512
+ flushRefresh(item);
10513
+ },
9471
10514
  inspect() { write({ type: "inspect", id }); },
9472
10515
  close() { write({ type: "unsubscribe", id }); subscriptions.delete(id); assembler.cancel(id); },
9473
10516
  } };
@@ -9508,6 +10551,7 @@ function createSshCollectorClient({ alias, createTransport = null, random = Math
9508
10551
  }
9509
10552
 
9510
10553
  function collectorDisplayDecision({ connected, hasSnapshot, hold = null }) {
10554
+ if (hold === "app-auth") return { disconnected: true, reason: "app-auth" };
9511
10555
  if (!connected || hold === "disconnected") return { disconnected: true };
9512
10556
  if (hold === "shared-wait") return { mode: "waiting", waitCause: "shared-lane" };
9513
10557
  if (["observer", "primary", "secondary", "coordination"].includes(hold)) {
@@ -9947,8 +10991,10 @@ const ACQUISITION_METRIC_KEYS = [
9947
10991
  const ACQUISITION_HOLD_REASONS = new Set([
9948
10992
  "observer", "primary", "secondary", "disconnected", "coordination",
9949
10993
  ]);
9950
- const ACQUISITION_STORE_VERSION = 1;
10994
+ const ACQUISITION_STORE_VERSION = 2;
9951
10995
  const ACQUISITION_MAX_UNCERTAIN_RECEIPTS = 1024;
10996
+ const ACQUISITION_MAX_QUOTA_PROJECTIONS = 128;
10997
+ const ACQUISITION_QUOTA_PROJECTION_TTL_MS = 60_000;
9952
10998
 
9953
10999
  function normalizeAcquisitionDiagnosticMetadata(raw, path) {
9954
11000
  const normalized = normalizeAcquisitionStore(raw);
@@ -10098,16 +11144,23 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10098
11144
  if (!admitted?.ok || admitted.value.status !== "started") {
10099
11145
  return { index, skipped: { ...skippedDoctorProbe(name, args, admitted), ...(document ? { document } : {}) } };
10100
11146
  }
10101
- return { index, name, args, operation, input, document, reservationId: admitted.value.reservationId };
11147
+ return { index, name, args, operation, input, document,
11148
+ reservationId: admitted.value.reservationId,
11149
+ receiptCapability: admitted.value.receiptCapability };
10102
11150
  });
10103
11151
  results = admittedProbes.map((item) => item.skipped ?? null);
10104
11152
  const runnable = admittedProbes.filter((item) => !item.skipped);
10105
11153
  const settled = await mapAllSettledBounded(runnable, 4, async (item) => {
10106
- const result = await requestIdentityStorage.run(scope, () => probe(item.name, item.args, item.operation, item.input));
11154
+ const result = await requestIdentityStorage.run({ ...scope,
11155
+ receiptCapability: item.receiptCapability }, () => probe(item.name, item.args, item.operation, item.input));
10107
11156
  if (item.document) result.document = item.document;
10108
- completeReservation({ ...scope, identityProvider: null }, item.reservationId, result.failed
11157
+ const terminal = queueTerminalization(scope, item.receiptCapability, result.failed
10109
11158
  ? { outcome: "rejected" }
10110
- : { outcome: "measured-success", actualCost: operationCost(item.operation) }, Date.now());
11159
+ : { outcome: "measured-success", actualCosts: operationCost(item.operation) });
11160
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
11161
+ result.failed = true;
11162
+ result.stderr = `receipt settlement ${terminal.status}`;
11163
+ }
10111
11164
  return result;
10112
11165
  });
10113
11166
  settled.forEach((outcome, index) => {
@@ -10128,9 +11181,32 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10128
11181
  governorResult = inspectGovernor(cleanup, Date.now());
10129
11182
  }
10130
11183
  }
11184
+ const cachedQuota = !probeEndpoints ? doctorCachedQuotaScope(effectiveHost) : null;
10131
11185
  const governor = probeEndpoints
10132
11186
  ? governorHealth(governorResult, Date.now())
10133
- : { status: "not inspected (--probe not requested)", leases: 0, resources: {} };
11187
+ : cachedQuota?.ok ? governorHealth(cachedQuota.result, Date.now())
11188
+ : { status: `unavailable (${cachedQuota?.reason ?? "cached scope unavailable"})`, leases: 0, resources: {} };
11189
+ const diagnosticScope = probeEndpoints
11190
+ ? scopeResult?.ok ? scopeResult.value : null
11191
+ : cachedQuota?.ok ? { path: cachedQuota.path } : null;
11192
+ const recoveryJournal = diagnosticScope ? readRecoveryTransitions(diagnosticScope) : null;
11193
+ const recoveryDetails = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
11194
+ diagnosticScope ? recoveryLedgerSummary(diagnosticScope, resource) : null]));
11195
+ const currentRecovery = RATE_RESOURCES.map((resource) => recoveryJournal?.current[resource])
11196
+ .filter(Boolean).sort((left, right) => right.at - left.at)[0] ?? null;
11197
+ const lastFailedRecovery = [...(recoveryJournal?.entries ?? [])].reverse()
11198
+ .find((entry) => entry.code !== "clear") ?? null;
11199
+ const retryDates = RATE_RESOURCES.flatMap((resource) => {
11200
+ const active = recoveryJournal?.current[resource];
11201
+ const observer = (probeEndpoints ? governorResult : cachedQuota?.result)?.value?.observers[resource];
11202
+ return [active?.retryAt, observer?.outcome === "failed" ? observer.nextAt : null];
11203
+ }).filter((value) => Number.isFinite(value) && value > Date.now());
11204
+ const nextRecoveryAt = retryDates.length > 0 ? Math.min(...retryDates) : null;
11205
+ let recoveryReason = "unavailable";
11206
+ if (currentRecovery) recoveryReason = `${currentRecovery.resource}: ${currentRecovery.code}`;
11207
+ else if (recoveryJournal?.status === "corrupt") recoveryReason = "diagnostic journal corrupt";
11208
+ else if (recoveryJournal?.status === "unavailable") recoveryReason = "diagnostic journal unavailable";
11209
+ else if (["healthy", "missing"].includes(recoveryJournal?.status)) recoveryReason = "none recorded";
10134
11210
  const diagnosticSource = runtime.connect === "local"
10135
11211
  ? "local collector"
10136
11212
  : runtime.connect?.startsWith("ssh:") ? "SSH collector" : "standalone";
@@ -10175,6 +11251,10 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10175
11251
  ),
10176
11252
  "",
10177
11253
  ...section("API governor"),
11254
+ field("cached scope", probeEndpoints ? "verified by probe" : cachedQuota?.ok ? "read-only cached identity" : `unavailable (${cachedQuota?.reason ?? "unknown"})`),
11255
+ ...(cachedQuota?.path && !cachedQuota.ok
11256
+ ? [field("state file", `${join(dirname(cachedQuota.path), "quota-[redacted].json")} (quota fingerprint withheld)`)]
11257
+ : []),
10178
11258
  field("status", governor.status),
10179
11259
  field("live leases", governor.leases),
10180
11260
  ...RATE_RESOURCES.map((resource) => {
@@ -10186,6 +11266,21 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10186
11266
  : "unavailable",
10187
11267
  );
10188
11268
  }),
11269
+ field("recovery reason", recoveryReason),
11270
+ field("last failed transition", lastFailedRecovery?.code ?? "unavailable"),
11271
+ ...RATE_RESOURCES.flatMap((resource) => [
11272
+ field(`${resource} receipt units`, recoveryDetails[resource]?.receiptUnits ?? "unavailable"),
11273
+ field(`${resource} receipt count`, recoveryDetails[resource]?.receiptCount ?? "unavailable"),
11274
+ field(`${resource} debt units`, recoveryDetails[resource]?.debtUnits ?? "unavailable"),
11275
+ field(`${resource} debt count`, recoveryDetails[resource]?.debtCount ?? "unavailable"),
11276
+ field(`${resource} oldest unresolved`, recoveryDetails[resource]?.oldestUnresolvedAt == null
11277
+ ? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].oldestUnresolvedAt))),
11278
+ field(`${resource} observer age`, recoveryDetails[resource]?.observerAt == null
11279
+ ? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].observerAt))),
11280
+ ]),
11281
+ field("next retry", nextRecoveryAt == null ? "unavailable" : new Date(nextRecoveryAt).toISOString()),
11282
+ field("terminalization backlog", currentRecovery?.terminalizationBacklog == null
11283
+ ? "unavailable" : `${currentRecovery.terminalizationBacklog} (last recorded)`),
10189
11284
  "",
10190
11285
  ...section("Acquisition metrics"),
10191
11286
  field("source", acquisitionDiagnostic?.source ?? "standalone"),
@@ -10202,7 +11297,8 @@ async function runDoctor({ probeEndpoints = false } = {}) {
10202
11297
  field("failed requests", acquisitionDiagnostic?.metrics.failedRequests ?? 0),
10203
11298
  field("REST actual 200/304", `${acquisitionDiagnostic?.metrics.rest200 ?? 0}/${acquisitionDiagnostic?.metrics.rest304 ?? 0}`),
10204
11299
  field("proven actual units", `${acquisitionDiagnostic?.metrics.coreUnits ?? 0} core + ${acquisitionDiagnostic?.metrics.graphqlUnits ?? 0} GraphQL`),
10205
- field("conservative outstanding", `${acquisitionDiagnostic?.metrics.uncertainCoreUnits ?? 0} core + ${acquisitionDiagnostic?.metrics.uncertainGraphqlUnits ?? 0} GraphQL`),
11300
+ field("conservative outstanding", acquisitionOutstandingText(acquisitionDiagnostic)),
11301
+ field("legacy unverified", `${acquisitionDiagnostic?.legacyUnverified?.coreUnits ?? 0} core + ${acquisitionDiagnostic?.legacyUnverified?.graphqlUnits ?? 0} GraphQL (diagnostic only)`),
10206
11302
  field("observer calls", acquisitionDiagnostic?.metrics.observerCalls ?? 0),
10207
11303
  field("cache/followers", `${acquisitionDiagnostic?.metrics.cacheHits ?? 0}/${acquisitionDiagnostic?.metrics.joinedFollowers ?? 0}`),
10208
11304
  field("queue wait (measured)", `${acquisitionDiagnostic?.metrics.queueWaitMs ?? 0}ms`),
@@ -10321,7 +11417,7 @@ function parseRepoTarget(value) {
10321
11417
  // in-flight guard absorbs every other one and the effective rate is whatever
10322
11418
  // `gh` can sustain -- the requested interval silently stops being real. Clamping
10323
11419
  // with a stated minimum is honest where silently accepting it would not be.
10324
- const BACKGROUND_MODES = ["all", "off"];
11420
+ const BACKGROUND_MODES = ["all", "off", "on-demand"];
10325
11421
  const MIN_REFRESH_SECONDS = 2;
10326
11422
  const MAX_REFRESH_SECONDS = 3600;
10327
11423
 
@@ -10551,7 +11647,11 @@ Options:
10551
11647
  and running Actions are checked every ${POLL_ACTIVE_CI_MS / 1000}s.
10552
11648
  Safe shared grants may run later.
10553
11649
  --tab <name> Tab to start on: ${TAB_KEYS.join(", ")}.
10554
- --background <mode> ${BACKGROUND_MODES.join(" or ")} (default all). \`off\` never requests data
11650
+ --background <mode> ${BACKGROUND_MODES.join(" or ")} (default on-demand).
11651
+ on-demand watches active Actions; other tabs refresh
11652
+ when opened or with r, then keep cached results.
11653
+ all polls every tab; off polls the selected tab.
11654
+ \`off\` never requests data
10555
11655
  for a tab you are not looking at; its count stays at
10556
11656
  the last known value and visibly ages.
10557
11657
  --verbose Write one line per gh invocation to stderr. stderr
@@ -10579,9 +11679,11 @@ Run it from inside a locally cloned GitHub repository; the repo is inferred
10579
11679
  from the git remote, the same way \`gh\` does it. Requires the \`gh\` CLI
10580
11680
  (2.20 or newer), authenticated via \`gh auth login\`.
10581
11681
 
10582
- On a healthy single pane, the active tab is considered every ${REFRESH_MS / 1000}s while its
11682
+ By default, active Actions is considered every ${REFRESH_MS / 1000}s while its
10583
11683
  content is changing, then every ${POLL_QUIET_MS.actions / 1000}s once two checks in a row come back
10584
- unchanged; each background tab is considered about every ${Math.max(REFRESH_MS * BACKGROUND_EVERY, POLL_BACKGROUND_MS.actions) / 1000}s. Quota-consuming
11684
+ unchanged. Secondary tabs check on opening or r and retain cached results.
11685
+ With --background all, inactive tabs are considered about every
11686
+ ${Math.max(REFRESH_MS * BACKGROUND_EVERY, POLL_BACKGROUND_MS.actions) / 1000}s (Security is slower). Quota-consuming
10585
11687
  calls still need a shared, resource-specific grant that preserves a hard reserve.
10586
11688
  Scheduled checks and manual refresh do not bypass that safety check: neither
10587
11689
  refresh key can spend past the reserve.
@@ -12326,6 +13428,8 @@ const ACQUISITION_MAX_ENTITY_BYTES = 1024 * 1024;
12326
13428
  const ACQUISITION_MAX_ENTITIES = 512;
12327
13429
  const ACQUISITION_MAX_LIVE_TARGETS = 32;
12328
13430
  const ACQUISITION_MAX_SUBSCRIPTIONS = 128;
13431
+ const ACQUISITION_READ_MAX_ATTEMPTS = 24;
13432
+ const ACQUISITION_READ_MAX_MS = 1_500;
12329
13433
  let acquisitionTempSequence = 0;
12330
13434
 
12331
13435
  function acquisitionStorePath(options = {}) {
@@ -12337,7 +13441,8 @@ function emptyAcquisitionStore() {
12337
13441
  version: ACQUISITION_STORE_VERSION,
12338
13442
  producerEpoch: governorId(),
12339
13443
  metrics: emptyAcquisitionMetrics(),
12340
- uncertainReceipts: {},
13444
+ quotaProjections: {},
13445
+ legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
12341
13446
  subscriptions: {},
12342
13447
  queries: {},
12343
13448
  aliases: {},
@@ -12365,32 +13470,6 @@ function acquisitionMetricState(state) {
12365
13470
  return state.metrics;
12366
13471
  }
12367
13472
 
12368
- function removeAcquisitionReceipt(state, id) {
12369
- const receipt = state.uncertainReceipts?.[id];
12370
- if (!receipt) return false;
12371
- const metrics = acquisitionMetricState(state);
12372
- const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
12373
- metrics[key] = Math.max(0, metrics[key] - receipt.units);
12374
- delete state.uncertainReceipts[id];
12375
- for (const record of Object.values(state.queries ?? {})) {
12376
- if (record.claim?.receiptIds?.includes(id)) {
12377
- record.claim.receiptIds = record.claim.receiptIds.filter((receiptId) => receiptId !== id);
12378
- }
12379
- }
12380
- return true;
12381
- }
12382
-
12383
- function addAcquisitionReceipt(state, receipt) {
12384
- state.uncertainReceipts ??= {};
12385
- if (state.uncertainReceipts[receipt.id]) return true;
12386
- if (Object.keys(state.uncertainReceipts).length >= ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return false;
12387
- state.uncertainReceipts[receipt.id] = receipt;
12388
- const metrics = acquisitionMetricState(state);
12389
- const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
12390
- metrics[key] += receipt.units;
12391
- return true;
12392
- }
12393
-
12394
13473
  function normalizeAcquisitionReceipt(raw, expectedId = raw?.id) {
12395
13474
  if (!isRecord(raw) || raw.id !== expectedId || typeof raw.id !== "string" || raw.id.length > 240 ||
12396
13475
  typeof raw.reservationId !== "string" || raw.reservationId.length < 1 ||
@@ -12415,63 +13494,6 @@ function addAcquisitionRequestMetrics(state, delta) {
12415
13494
  return addAcquisitionMetrics(state, observed);
12416
13495
  }
12417
13496
 
12418
- function releaseAcquisitionUncertainty(state, record) {
12419
- if (!record?.claim?.started) return;
12420
- const receiptIds = record.claim.receiptIds ?? [];
12421
- if (Object.hasOwn(record.claim, "receiptIds")) {
12422
- for (const id of receiptIds) removeAcquisitionReceipt(state, id);
12423
- return;
12424
- }
12425
- const costs = tabRequestCost(record.query.resource);
12426
- const metrics = acquisitionMetricState(state);
12427
- metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - costs.core);
12428
- metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - costs.graphql);
12429
- }
12430
-
12431
- function releaseKnownAcquisitionUncertainty(state, record, requestMetrics) {
12432
- if (!record?.claim?.started) return;
12433
- const receiptIds = record.claim.receiptIds ?? [];
12434
- if (Object.hasOwn(record.claim, "receiptIds")) {
12435
- for (const resource of RATE_RESOURCES) {
12436
- const metricKey = resource === "core" ? "coreUnits" : "graphqlUnits";
12437
- const uncertainKey = resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
12438
- const resourceReceipts = receiptIds
12439
- .map((id) => state.uncertainReceipts?.[id])
12440
- .filter((receipt) => receipt?.resource === resource);
12441
- if (Object.hasOwn(requestMetrics, uncertainKey)) {
12442
- let remaining = requestMetrics[uncertainKey];
12443
- for (const receipt of resourceReceipts) {
12444
- const retained = Math.min(receipt.units, remaining);
12445
- const released = receipt.units - retained;
12446
- remaining -= retained;
12447
- if (retained === 0) removeAcquisitionReceipt(state, receipt.id);
12448
- else if (released > 0) {
12449
- receipt.units = retained;
12450
- acquisitionMetricState(state)[uncertainKey] -= released;
12451
- }
12452
- }
12453
- continue;
12454
- }
12455
- if (Object.hasOwn(requestMetrics, metricKey)) {
12456
- for (const receipt of resourceReceipts) removeAcquisitionReceipt(state, receipt.id);
12457
- }
12458
- }
12459
- return;
12460
- }
12461
- const declared = tabRequestCost(record.query.resource);
12462
- const metrics = acquisitionMetricState(state);
12463
- if (Object.hasOwn(requestMetrics, "uncertainCoreUnits")) {
12464
- metrics.uncertainCoreUnits = Math.min(declared.core, requestMetrics.uncertainCoreUnits);
12465
- } else if (Object.hasOwn(requestMetrics, "coreUnits")) {
12466
- metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - declared.core);
12467
- }
12468
- if (Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
12469
- metrics.uncertainGraphqlUnits = Math.min(declared.graphql, requestMetrics.uncertainGraphqlUnits);
12470
- } else if (Object.hasOwn(requestMetrics, "graphqlUnits")) {
12471
- metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - declared.graphql);
12472
- }
12473
- }
12474
-
12475
13497
  function settleAcquisitionWaiters(state, queryKey, generation, at) {
12476
13498
  let queueWaitMs = 0;
12477
13499
  for (const subscription of Object.values(state.subscriptions)) {
@@ -12490,8 +13512,6 @@ function settleInvalidAcquisitionPublication(state, record, claim, requestMetric
12490
13512
  ...normalized,
12491
13513
  failedRequests: Math.max(1, normalized.failedRequests ?? 0),
12492
13514
  });
12493
- releaseKnownAcquisitionUncertainty(state, record, normalized);
12494
- for (const receipt of receipts ?? []) addAcquisitionReceipt(state, receipt);
12495
13515
  settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, at);
12496
13516
  record.claim = null;
12497
13517
  record.hold = { reason: "primary", at };
@@ -12528,9 +13548,18 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
12528
13548
  subscriptionsByQuery.set(subscription.queryKey, grouped);
12529
13549
  }
12530
13550
  const activeQueryKeys = new Set(subscriptions.map((subscription) => subscription.queryKey));
13551
+ const outstandingQuota = Object.entries(state?.quotaProjections ?? {}).map(([scopeHash, projection]) => {
13552
+ const current = projection.status === "current" && projection.at <= nowMs &&
13553
+ nowMs - projection.at <= ACQUISITION_QUOTA_PROJECTION_TTL_MS;
13554
+ return { scopeHash, revision: projection.revision,
13555
+ status: current ? "current" : "unavailable", units: current ? projection.units : null,
13556
+ at: projection.at, accessKeys: projection.accessKeys };
13557
+ });
12531
13558
  const queries = Object.values(state?.queries ?? {}).map((record) => {
12532
13559
  const snapshot = record.snapshot;
12533
13560
  const consumers = subscriptionsByQuery.get(record.query.queryKey) ?? [];
13561
+ const projection = outstandingQuota.find((item) =>
13562
+ item.accessKeys.includes(record.query.accessKey) && item.status === "current");
12534
13563
  return {
12535
13564
  resource: record.query.resource,
12536
13565
  lastSuccessAt: snapshot?.lastSuccessAt ?? null,
@@ -12543,8 +13572,9 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
12543
13572
  graphql: state?.metrics?.graphqlUnits ?? 0,
12544
13573
  },
12545
13574
  uncertainCost: {
12546
- core: state?.metrics?.uncertainCoreUnits ?? 0,
12547
- graphql: state?.metrics?.uncertainGraphqlUnits ?? 0,
13575
+ status: projection ? "current" : "unavailable",
13576
+ core: projection?.units.core ?? null,
13577
+ graphql: projection?.units.graphql ?? null,
12548
13578
  },
12549
13579
  coalescedConsumers: consumers.length,
12550
13580
  };
@@ -12556,12 +13586,31 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
12556
13586
  activeQueries: activeQueryKeys.size,
12557
13587
  activeSubscribers: subscriptions.length,
12558
13588
  metrics: normalizeAcquisitionMetrics(state?.metrics) ?? emptyAcquisitionMetrics(),
13589
+ outstandingQuota: outstandingQuota.map(({ accessKeys: _accessKeys, ...projection }) => projection),
13590
+ legacyUnverified: state?.legacyUnverified ?? { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
12559
13591
  queries,
12560
13592
  };
12561
13593
  }
12562
13594
 
13595
+ function acquisitionOutstandingText(diagnostic) {
13596
+ const scopes = diagnostic?.outstandingQuota ?? [];
13597
+ if (scopes.length === 0 || scopes.some((scope) => scope.status !== "current")) {
13598
+ return "unavailable (quota projection pending)";
13599
+ }
13600
+ const units = scopes.reduce((total, scope) => ({
13601
+ core: total.core + scope.units.core,
13602
+ graphql: total.graphql + scope.units.graphql,
13603
+ }), { core: 0, graphql: 0 });
13604
+ return Number.isSafeInteger(units.core) && Number.isSafeInteger(units.graphql)
13605
+ ? `${units.core} core + ${units.graphql} GraphQL across ${scopes.length} quota scopes`
13606
+ : "unavailable (accounting overflow)";
13607
+ }
13608
+
12563
13609
  function acquisitionFailureHold(error) {
12564
13610
  if (typeof error?.providerCapability === "string") return "observer";
13611
+ if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) {
13612
+ return HTTP_PERMIT_FAILURE_HOLDS[error.coordinationReason];
13613
+ }
12565
13614
  if (SECONDARY_LIMIT_PATTERN.test(errText(error))) return "secondary";
12566
13615
  if (isRateLimited(error) || /API budget paused/i.test(errText(error))) return "primary";
12567
13616
  const attemptedHttp = Number.isSafeInteger(error?.requestMetrics?.httpRequests) &&
@@ -12889,32 +13938,58 @@ function prepareAcquisitionSnapshot(queryKey, snapshot) {
12889
13938
  }
12890
13939
 
12891
13940
  function normalizeAcquisitionStore(raw) {
12892
- if (!isRecord(raw) || raw.version !== ACQUISITION_STORE_VERSION || !validGovernorId(raw.producerEpoch) ||
13941
+ if (!isRecord(raw) || ![1, ACQUISITION_STORE_VERSION].includes(raw.version) || !validGovernorId(raw.producerEpoch) ||
12893
13942
  !isRecord(raw.subscriptions) || !isRecord(raw.queries) || !isRecord(raw.aliases)) return null;
13943
+ const legacy = raw.version === 1;
12894
13944
  const state = {
12895
13945
  version: ACQUISITION_STORE_VERSION,
12896
- producerEpoch: raw.producerEpoch,
13946
+ producerEpoch: legacy ? governorId() : raw.producerEpoch,
12897
13947
  metrics: normalizeAcquisitionMetrics(raw.metrics),
12898
- uncertainReceipts: {},
13948
+ quotaProjections: {},
13949
+ legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
12899
13950
  subscriptions: {},
12900
13951
  queries: {},
12901
13952
  aliases: {},
12902
13953
  };
12903
13954
  if (state.metrics === null) return null;
12904
- const receipts = raw.uncertainReceipts ?? {};
12905
- if (!isRecord(receipts) || Object.keys(receipts).length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return null;
12906
- for (const [id, receipt] of Object.entries(receipts)) {
12907
- const normalized = normalizeAcquisitionReceipt(receipt, id);
12908
- if (!normalized) return null;
12909
- state.uncertainReceipts[id] = normalized;
12910
- }
12911
- if (raw.uncertainReceipts !== undefined) {
12912
- const outstanding = Object.values(state.uncertainReceipts).reduce((total, receipt) => {
12913
- total[receipt.resource] += receipt.units;
12914
- return total;
12915
- }, { core: 0, graphql: 0 });
12916
- if (state.metrics.uncertainCoreUnits !== outstanding.core ||
12917
- state.metrics.uncertainGraphqlUnits !== outstanding.graphql) return null;
13955
+ if (legacy) {
13956
+ const receipts = raw.uncertainReceipts ?? {};
13957
+ if (!isRecord(receipts) || Object.keys(receipts).length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return null;
13958
+ for (const [id, receipt] of Object.entries(receipts)) {
13959
+ const normalized = normalizeAcquisitionReceipt(receipt, id);
13960
+ if (!normalized) return null;
13961
+ const key = receipt.resource === "core" ? "coreUnits" : "graphqlUnits";
13962
+ const next = state.legacyUnverified[key] + receipt.units;
13963
+ if (!Number.isSafeInteger(next)) return null;
13964
+ state.legacyUnverified[key] = next;
13965
+ state.legacyUnverified.receiptCount += 1;
13966
+ }
13967
+ if (raw.uncertainReceipts === undefined) {
13968
+ state.legacyUnverified.coreUnits = state.metrics.uncertainCoreUnits;
13969
+ state.legacyUnverified.graphqlUnits = state.metrics.uncertainGraphqlUnits;
13970
+ } else if (state.metrics.uncertainCoreUnits !== state.legacyUnverified.coreUnits ||
13971
+ state.metrics.uncertainGraphqlUnits !== state.legacyUnverified.graphqlUnits) return null;
13972
+ state.metrics.uncertainCoreUnits = 0;
13973
+ state.metrics.uncertainGraphqlUnits = 0;
13974
+ } else {
13975
+ if (!isRecord(raw.quotaProjections) || !isRecord(raw.legacyUnverified) ||
13976
+ Object.keys(raw.quotaProjections).length > ACQUISITION_MAX_QUOTA_PROJECTIONS ||
13977
+ !exactKeys(raw.legacyUnverified, ["coreUnits", "graphqlUnits", "receiptCount"]) ||
13978
+ Object.values(raw.legacyUnverified).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
13979
+ state.legacyUnverified = { ...raw.legacyUnverified };
13980
+ for (const [scopeHash, projection] of Object.entries(raw.quotaProjections)) {
13981
+ if (!/^[0-9a-f]{64}$/.test(scopeHash) || !exactKeys(projection, ["revision", "units", "at", "status", "accessKeys"]) ||
13982
+ !Number.isSafeInteger(projection.revision) || projection.revision < 0 ||
13983
+ !exactKeys(projection.units, RATE_RESOURCES) ||
13984
+ Object.values(projection.units).some((value) => !Number.isSafeInteger(value) || value < 0) ||
13985
+ !Number.isFinite(projection.at) || projection.at < 0 ||
13986
+ !["current", "unavailable"].includes(projection.status) ||
13987
+ !Array.isArray(projection.accessKeys) || projection.accessKeys.length > ACQUISITION_MAX_SUBSCRIPTIONS ||
13988
+ new Set(projection.accessKeys).size !== projection.accessKeys.length ||
13989
+ projection.accessKeys.some((key) => !/^[0-9a-f]{64}$/.test(key))) return null;
13990
+ state.quotaProjections[scopeHash] = { ...projection, units: { ...projection.units },
13991
+ accessKeys: [...projection.accessKeys] };
13992
+ }
12918
13993
  }
12919
13994
  for (const [key, value] of Object.entries(raw.aliases)) {
12920
13995
  if (typeof key !== "string" || typeof value !== "string") return null;
@@ -12933,11 +14008,15 @@ function normalizeAcquisitionStore(raw) {
12933
14008
  candidate.generation !== value.generation + 1 || !Number.isFinite(candidate.claimedAt) ||
12934
14009
  !Number.isFinite(candidate.leaseUntil) || candidate.leaseUntil < candidate.claimedAt ||
12935
14010
  typeof candidate.started !== "boolean") return null;
12936
- const hasReceiptIds = Object.hasOwn(candidate, "receiptIds");
12937
- const receiptIds = candidate.receiptIds ?? [];
12938
- if (!Array.isArray(receiptIds) || receiptIds.some((id) => typeof id !== "string" ||
12939
- !state.uncertainReceipts[id])) return null;
12940
- claim = { ...candidate, ...(hasReceiptIds ? { receiptIds } : {}) };
14011
+ if (legacy) {
14012
+ const receipts = raw.uncertainReceipts ?? {};
14013
+ const receiptIds = candidate.receiptIds ?? [];
14014
+ if (!Array.isArray(receiptIds) || receiptIds.some((id) => typeof id !== "string" || !receipts[id])) return null;
14015
+ claim = null; // v1 producer authority is fenced by the new epoch.
14016
+ } else {
14017
+ if (Object.hasOwn(candidate, "receiptIds")) return null;
14018
+ claim = { ...candidate };
14019
+ }
12941
14020
  }
12942
14021
  const snapshot = value.snapshot === null ? null
12943
14022
  : value.snapshot?.artifact
@@ -12990,37 +14069,63 @@ function loadAcquisitionMetadata(path) {
12990
14069
  }
12991
14070
  }
12992
14071
 
12993
- function readAcquisitionSnapshot(path, record) {
12994
- if (!record?.snapshot) return { ok: true, value: null };
12995
- if (Array.isArray(record.snapshot.rows)) return { ok: true, value: record.snapshot };
12996
- try {
12997
- const raw = readFileSync(join(acquisitionSnapshotRoot(path), record.snapshot.artifact), "utf8");
12998
- if (Buffer.byteLength(raw) !== record.snapshot.bytes ||
12999
- createHash("sha256").update(raw).digest("hex") !== record.snapshot.digest) {
13000
- return { ok: false, reason: "corrupt" };
14072
+ function readAcquisitionSnapshot(path, record, { deadline = performance.now() + ACQUISITION_READ_MAX_MS } = {}) {
14073
+ let current = record;
14074
+ for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
14075
+ performance.now() < deadline; attempt += 1) {
14076
+ if (!current?.snapshot) return current === record ? { ok: true, value: null }
14077
+ : { ok: false, reason: "stale" };
14078
+ if (Array.isArray(current.snapshot.rows)) {
14079
+ return { ok: true, value: current.snapshot, replaced: current !== record };
14080
+ }
14081
+ try {
14082
+ const raw = readFileSync(join(acquisitionSnapshotRoot(path), current.snapshot.artifact), "utf8");
14083
+ if (Buffer.byteLength(raw) !== current.snapshot.bytes ||
14084
+ createHash("sha256").update(raw).digest("hex") !== current.snapshot.digest) {
14085
+ return { ok: false, reason: "corrupt" };
14086
+ }
14087
+ const parsed = JSON.parse(raw);
14088
+ const snapshot = normalizeAcquisitionSnapshot(
14089
+ parsed, current.query, current.generation, parsed.producerEpoch,
14090
+ );
14091
+ return snapshot ? { ok: true, value: snapshot, replaced: current !== record }
14092
+ : { ok: false, reason: "corrupt" };
14093
+ } catch (error) {
14094
+ if (error?.code !== "ENOENT") return { ok: false, reason: "corrupt", error };
14095
+ const latest = loadAcquisitionMetadata(path);
14096
+ if (!latest.ok) return latest;
14097
+ const replacement = latest.value.queries[current.query.queryKey];
14098
+ if (!replacement) return { ok: false, reason: "stale" };
14099
+ if (replacement.snapshot?.artifact === current.snapshot.artifact) {
14100
+ return { ok: false, reason: "corrupt", error };
14101
+ }
14102
+ current = replacement;
13001
14103
  }
13002
- const parsed = JSON.parse(raw);
13003
- const snapshot = normalizeAcquisitionSnapshot(
13004
- parsed, record.query, record.generation, parsed.producerEpoch,
13005
- );
13006
- return snapshot ? { ok: true, value: snapshot } : { ok: false, reason: "corrupt" };
13007
- } catch (error) {
13008
- return { ok: false, reason: "corrupt", error };
13009
14104
  }
14105
+ return { ok: false, reason: "busy" };
13010
14106
  }
13011
14107
 
13012
- function hydrateAcquisitionStore(path, state) {
14108
+ function hydrateAcquisitionStore(path, state, deadline) {
13013
14109
  for (const record of Object.values(state.queries)) {
13014
- const loaded = readAcquisitionSnapshot(path, record);
14110
+ const loaded = readAcquisitionSnapshot(path, record, { deadline });
13015
14111
  if (!loaded.ok) return loaded;
14112
+ if (loaded.replaced) return { ok: false, reason: "stale" };
13016
14113
  record.snapshot = loaded.value;
13017
14114
  }
13018
14115
  return { ok: true, value: state };
13019
14116
  }
13020
14117
 
13021
- function loadAcquisitionStore(path) {
13022
- const loaded = loadAcquisitionMetadata(path);
13023
- return loaded.ok ? hydrateAcquisitionStore(path, loaded.value) : loaded;
14118
+ function loadAcquisitionStore(path, { onMetadataLoaded = null } = {}) {
14119
+ const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
14120
+ for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
14121
+ performance.now() < deadline; attempt += 1) {
14122
+ const loaded = loadAcquisitionMetadata(path);
14123
+ if (!loaded.ok) return loaded;
14124
+ onMetadataLoaded?.();
14125
+ const hydrated = hydrateAcquisitionStore(path, loaded.value, deadline);
14126
+ if (hydrated.reason !== "stale") return hydrated;
14127
+ }
14128
+ return { ok: false, reason: "busy" };
13024
14129
  }
13025
14130
 
13026
14131
  function writeAcquisitionArtifact(root, artifact, payload) {
@@ -13193,8 +14298,8 @@ function trimAcquisitionStore(state, protectedQueryKey = null) {
13193
14298
 
13194
14299
  async function runStartedAcquisitionTransport(markStarted, transport) {
13195
14300
  const started = await markStarted();
13196
- if (!started.ok) return started;
13197
- if (started.value?.status !== "started") return { ok: false, reason: "already-started" };
14301
+ if (!started.ok) return { ...started, notStarted: true };
14302
+ if (started.value?.status !== "started") return { ok: false, reason: "already-started", notStarted: true };
13198
14303
  return { ok: true, value: await transport() };
13199
14304
  }
13200
14305
 
@@ -13244,7 +14349,7 @@ function createAcquisitionEngine({
13244
14349
  }
13245
14350
  const loaded = loadRecordSnapshot(record);
13246
14351
  if (!loaded.ok) return loaded;
13247
- item.generation = record.generation;
14352
+ item.generation = loaded.value.generation;
13248
14353
  try { item.callback?.(loaded.value); } catch { /* subscriber callback is isolated */ }
13249
14354
  return { ok: true, value: true, snapshot: loaded.value };
13250
14355
  }
@@ -13277,7 +14382,8 @@ function createAcquisitionEngine({
13277
14382
  const record = loaded.value.queries[item.queryKey] ?? null;
13278
14383
  if (!record?.snapshot) return { ok: true, value: record };
13279
14384
  const snapshot = loadRecordSnapshot(record);
13280
- return snapshot.ok ? { ok: true, value: { ...record, snapshot: snapshot.value } } : snapshot;
14385
+ return snapshot.ok ? { ok: true, value: { ...record,
14386
+ generation: snapshot.value?.generation ?? record.generation, snapshot: snapshot.value } } : snapshot;
13281
14387
  }
13282
14388
 
13283
14389
  function heartbeat() {
@@ -13445,14 +14551,38 @@ function createAcquisitionEngine({
13445
14551
  aliasConflict: matching.length > 1 } };
13446
14552
  });
13447
14553
  if (!result.ok) return result;
13448
- const record = result.state.queries[result.value.queryKey];
13449
- const loadedSnapshot = loadRecordSnapshot(record);
13450
- if (!loadedSnapshot.ok) return loadedSnapshot;
13451
- result.value.snapshot = loadedSnapshot.value;
13452
- local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation: record?.generation ?? 0,
14554
+ // A publisher can canonicalize the query and remap this just-committed
14555
+ // shared subscription before its snapshot is hydrated. Register locally
14556
+ // first so every failure still has a cleanup owner, then follow the
14557
+ // subscription ID in fresh metadata rather than the old query key.
14558
+ local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation: 0,
13453
14559
  claimNonce: null, pending: null });
13454
14560
  ensureTimer();
13455
- return result;
14561
+ const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
14562
+ for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
14563
+ performance.now() < deadline; attempt += 1) {
14564
+ const latest = load();
14565
+ if (!latest.ok) { unsubscribe(id); return latest; }
14566
+ const shared = latest.value.subscriptions[id];
14567
+ if (!shared || shared.nonce !== nonce) { unsubscribe(id); return { ok: false, reason: "stale" }; }
14568
+ const item = local.get(id);
14569
+ item.queryKey = shared.queryKey;
14570
+ const record = latest.value.queries[shared.queryKey];
14571
+ if (!record) continue;
14572
+ const loadedSnapshot = loadRecordSnapshot(record);
14573
+ if (!loadedSnapshot.ok) {
14574
+ if (["stale", "busy"].includes(loadedSnapshot.reason)) continue;
14575
+ unsubscribe(id);
14576
+ return loadedSnapshot;
14577
+ }
14578
+ if (loadedSnapshot.replaced) continue;
14579
+ result.value.queryKey = shared.queryKey;
14580
+ result.value.snapshot = loadedSnapshot.value;
14581
+ item.generation = loadedSnapshot.value?.generation ?? record.generation;
14582
+ return result;
14583
+ }
14584
+ unsubscribe(id);
14585
+ return { ok: false, reason: "busy" };
13456
14586
  }
13457
14587
 
13458
14588
  function updateDemand(id, demandInput) {
@@ -13509,11 +14639,6 @@ function createAcquisitionEngine({
13509
14639
  const requestMetrics = produced?.requestMetrics === undefined
13510
14640
  ? {}
13511
14641
  : normalizeAcquisitionMetrics(produced.requestMetrics, { partial: true });
13512
- const receiptInputs = produced?.uncertainReceipts ?? [];
13513
- const receipts = Array.isArray(receiptInputs)
13514
- ? receiptInputs.map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
13515
- const receiptsValid = receipts && !receipts.includes(null) &&
13516
- receipts.every((receipt) => receipt.accessKey === record.query.accessKey);
13517
14642
  const loadedPrevious = loadRecordSnapshot(record);
13518
14643
  if (!loadedPrevious.ok) return loadedPrevious;
13519
14644
  if (loadedPrevious.value) record.snapshot = loadedPrevious.value;
@@ -13527,20 +14652,15 @@ function createAcquisitionEngine({
13527
14652
  snapshot = { ...snapshot, lastChangedAt: loadedPrevious.value.lastChangedAt };
13528
14653
  }
13529
14654
  if (!snapshot) {
13530
- settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics, receiptsValid ? receipts : [], now());
14655
+ settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics, [], now());
13531
14656
  return { value: { ok: false, reason: "invalid" } };
13532
14657
  }
13533
14658
  const previous = record.snapshot;
13534
14659
  const previousClaim = record.claim;
13535
- if (requestMetrics === null || !receiptsValid) {
14660
+ if (requestMetrics === null) {
13536
14661
  settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics ?? {}, [], now());
13537
14662
  return { value: { ok: false, reason: "invalid" } };
13538
14663
  }
13539
- const receiptIds = new Set(Object.keys(state.uncertainReceipts ?? {}));
13540
- const novelReceipts = receipts.filter((receipt) => !receiptIds.has(receipt.id));
13541
- if (receiptIds.size + novelReceipts.length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) {
13542
- return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
13543
- }
13544
14664
  record.generation = effectiveClaim.generation;
13545
14665
  record.snapshot = snapshot;
13546
14666
  record.claim = null;
@@ -13552,14 +14672,7 @@ function createAcquisitionEngine({
13552
14672
  record.claim = previousClaim;
13553
14673
  return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
13554
14674
  }
13555
- if (Object.hasOwn(requestMetrics, "uncertainCoreUnits") ||
13556
- Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
13557
- releaseKnownAcquisitionUncertainty(state, { ...record, claim: previousClaim }, requestMetrics);
13558
- } else {
13559
- releaseAcquisitionUncertainty(state, { ...record, claim: previousClaim });
13560
- }
13561
14675
  addAcquisitionRequestMetrics(state, requestMetrics);
13562
- for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
13563
14676
  settleAcquisitionWaiters(state, record.query.queryKey, effectiveClaim.generation, now());
13564
14677
  return { value: { ok: true, value: { role: "producer", snapshot,
13565
14678
  queryKey: record.query.queryKey, remapped } } };
@@ -13613,10 +14726,8 @@ function createAcquisitionEngine({
13613
14726
  function settleFailure(id, claim, failure) {
13614
14727
  const item = local.get(id);
13615
14728
  const requestMetrics = normalizeAcquisitionMetrics(failure?.requestMetrics ?? {}, { partial: true });
13616
- const receipts = Array.isArray(failure?.uncertainReceipts ?? [])
13617
- ? (failure?.uncertainReceipts ?? []).map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
13618
14729
  const hold = failure?.hold ?? "primary";
13619
- if (!item || !isRecord(claim) || requestMetrics === null || !receipts || receipts.includes(null) ||
14730
+ if (!item || !isRecord(claim) || requestMetrics === null ||
13620
14731
  !ACQUISITION_HOLD_REASONS.has(hold)) {
13621
14732
  return { ok: false, reason: "invalid" };
13622
14733
  }
@@ -13625,13 +14736,7 @@ function createAcquisitionEngine({
13625
14736
  if (!record || record.claim?.nonce !== claim.nonce || record.claim.generation !== claim.generation) {
13626
14737
  return { value: { ok: false, reason: "stale", definitive: true } };
13627
14738
  }
13628
- if (receipts.some((receipt) => receipt.accessKey !== record.query.accessKey) ||
13629
- Object.keys(state.uncertainReceipts ?? {}).length +
13630
- receipts.filter((receipt) => !state.uncertainReceipts?.[receipt.id]).length >
13631
- ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return { ok: false, reason: "capacity" };
13632
14739
  addAcquisitionRequestMetrics(state, requestMetrics);
13633
- releaseKnownAcquisitionUncertainty(state, record, requestMetrics);
13634
- for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
13635
14740
  settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, now());
13636
14741
  record.claim = null;
13637
14742
  record.hold = { reason: hold, at: now() };
@@ -13654,7 +14759,11 @@ function createAcquisitionEngine({
13654
14759
  const item = local.get(id);
13655
14760
  if (!item || reason !== null && !ACQUISITION_HOLD_REASONS.has(reason) ||
13656
14761
  expected !== null && (!isRecord(expected) || !TAB_KEYS.includes(expected.resource) ||
13657
- typeof expected.accessKey !== "string" || expected.accessKey.length === 0)) {
14762
+ typeof expected.accessKey !== "string" || expected.accessKey.length === 0 ||
14763
+ expected.currentHold !== undefined &&
14764
+ (!isRecord(expected.currentHold) ||
14765
+ !ACQUISITION_HOLD_REASONS.has(expected.currentHold.reason) ||
14766
+ !Number.isFinite(expected.currentHold.at)))) {
13658
14767
  return { ok: false, reason: "invalid" };
13659
14768
  }
13660
14769
  return transact((state) => {
@@ -13662,6 +14771,11 @@ function createAcquisitionEngine({
13662
14771
  if (!record) return { ok: false, reason: "stale" };
13663
14772
  if (expected && (record.query.resource !== expected.resource ||
13664
14773
  record.query.accessKey !== expected.accessKey)) return { ok: false, reason: "stale-access" };
14774
+ if (expected?.currentHold !== undefined &&
14775
+ (record.hold?.reason !== expected.currentHold.reason ||
14776
+ record.hold?.at !== expected.currentHold.at)) {
14777
+ return { ok: false, reason: "stale-hold" };
14778
+ }
13665
14779
  if ((record.hold?.reason ?? null) === reason) return { changed: false, value: true };
13666
14780
  const next = reason === null ? null : { reason, at: now() };
13667
14781
  record.hold = next;
@@ -13693,28 +14807,10 @@ function createAcquisitionEngine({
13693
14807
  if (now() >= acquisitionClaimDeadline(record.claim)) {
13694
14808
  return { ok: false, reason: "stale" };
13695
14809
  }
13696
- const receiptIds = [];
13697
- for (const resource of RATE_RESOURCES) {
13698
- if (costs[resource] <= 0) continue;
13699
- const epoch = typeof supplied?.epochs?.[resource] === "string" && supplied.epochs[resource].length > 0
13700
- ? supplied.epochs[resource] : "unknown";
13701
- const receipt = {
13702
- id: `${reservationId}:${resource}`,
13703
- reservationId,
13704
- accessKey: record.query.accessKey,
13705
- resource,
13706
- epoch,
13707
- units: costs[resource],
13708
- startedAt: now(),
13709
- };
13710
- if (!addAcquisitionReceipt(state, receipt)) return { ok: false, reason: "capacity" };
13711
- receiptIds.push(receipt.id);
13712
- }
13713
14810
  record.claim.started = true;
13714
14811
  record.claim.startedAt = now();
13715
14812
  record.claim.reservationId = reservationId;
13716
14813
  record.claim.startReceipt = startReceipt;
13717
- record.claim.receiptIds = receiptIds;
13718
14814
  return { value: { status: "started", reservationId } };
13719
14815
  });
13720
14816
  }
@@ -13764,7 +14860,8 @@ function createAcquisitionEngine({
13764
14860
  const claimed = transact((state) => {
13765
14861
  const reaped = reapAcquisitionSubscriptions(state, now(), kill);
13766
14862
  const subscription = state.subscriptions[id];
13767
- const record = state.queries[item.queryKey];
14863
+ const queryKey = subscription?.queryKey;
14864
+ const record = state.queries[queryKey];
13768
14865
  if (!subscription || subscription.nonce !== item.nonce || !record) return { ok: false, reason: "stale" };
13769
14866
  const requestedGeneration = force
13770
14867
  ? Math.max(subscription.requestedGeneration,
@@ -13776,8 +14873,8 @@ function createAcquisitionEngine({
13776
14873
  }
13777
14874
  if (requestedGeneration <= record.generation && record.snapshot?.nextDueAt > now()) {
13778
14875
  return { changed: reaped > 0 || demandChanged,
13779
- value: { role: "follower", reason: "fresh", snapshot: record.snapshot,
13780
- sharingCount: acquisitionSharingCount(state, item.queryKey) } };
14876
+ value: { role: "follower", reason: "fresh", queryKey, snapshot: record.snapshot,
14877
+ sharingCount: acquisitionSharingCount(state, queryKey) } };
13781
14878
  }
13782
14879
  if (record.claim) {
13783
14880
  const absoluteDeadline = acquisitionClaimDeadline(record.claim);
@@ -13792,10 +14889,10 @@ function createAcquisitionEngine({
13792
14889
  }
13793
14890
  return { changed: reaped > 0 || demandChanged || joined,
13794
14891
  value: { role: "follower", reason: expired ? `owner-${owner}` : "claimed",
13795
- snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state, item.queryKey) } };
14892
+ queryKey, snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state, queryKey) } };
13796
14893
  }
13797
14894
  }
13798
- const nonce = createId("acquisition-claim", item.queryKey, record.generation + 1);
14895
+ const nonce = createId("acquisition-claim", queryKey, record.generation + 1);
13799
14896
  record.claim = {
13800
14897
  pid,
13801
14898
  nonce,
@@ -13809,11 +14906,13 @@ function createAcquisitionEngine({
13809
14906
  subscription.waitingSinceAt = null;
13810
14907
  record.lastUsedAt = now();
13811
14908
  return { value: { role: "producer", nonce, generation: record.claim.generation,
13812
- query: record.query, snapshot: record.snapshot, demand: aggregateAcquisitionDemand(state, item.queryKey),
13813
- sharingCount: acquisitionSharingCount(state, item.queryKey) } };
14909
+ queryKey, query: record.query, snapshot: record.snapshot,
14910
+ demand: aggregateAcquisitionDemand(state, queryKey),
14911
+ sharingCount: acquisitionSharingCount(state, queryKey) } };
13814
14912
  });
14913
+ if (claimed.ok && claimed.value?.queryKey) item.queryKey = claimed.value.queryKey;
13815
14914
  if (claimed.ok && claimed.value?.snapshot) {
13816
- const record = claimed.state.queries[item.queryKey];
14915
+ const record = claimed.state.queries[claimed.value.queryKey];
13817
14916
  const loadedSnapshot = loadRecordSnapshot(record);
13818
14917
  if (!loadedSnapshot.ok) return loadedSnapshot;
13819
14918
  claimed.value.snapshot = loadedSnapshot.value;
@@ -13921,21 +15020,58 @@ function createAcquisitionEngine({
13921
15020
  });
13922
15021
  }
13923
15022
 
13924
- function reconcileUncertainty(accessKey, evidence) {
13925
- if (typeof accessKey !== "string" || accessKey.length === 0 || !isRecord(evidence)) {
15023
+ function syncQuotaProjection(scope) {
15024
+ if (!isRecord(scope) || !/^[0-9a-f]{64}$/.test(scope.hash ?? "")) {
13926
15025
  return { ok: false, reason: "invalid" };
13927
15026
  }
15027
+ // Quota is read before the acquisition lock. This diagnostic copy never
15028
+ // grants requests and a failed read cannot block source publication.
15029
+ const observed = inspectGovernor(scope, now());
15030
+ if (!observed.ok) {
15031
+ return transact((state) => {
15032
+ const previous = state.quotaProjections[scope.hash];
15033
+ if (!previous || previous.status === "unavailable") return { changed: false, value: false };
15034
+ previous.status = "unavailable";
15035
+ return { value: false };
15036
+ });
15037
+ }
15038
+ const ledger = observed.value;
15039
+ const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
15040
+ Object.values(ledger.reservations).reduce((sum, reservation) =>
15041
+ sum + reservationCost(reservation, resource, ledger.leases, now()),
15042
+ ledger.debt[resource].unresolvedUnits + ledger.debt[resource].quiescentUnits +
15043
+ reservationCost(ledger.controlReceipts[resource], resource, ledger.leases, now()))]));
15044
+ if (Object.values(units).some((value) => !Number.isSafeInteger(value) || value < 0)) {
15045
+ return { ok: false, reason: "accounting-overflow" };
15046
+ }
13928
15047
  return transact((state) => {
13929
- let changed = false;
13930
- for (const [id, receipt] of Object.entries(state.uncertainReceipts ?? {})) {
13931
- const observed = evidence[receipt.resource];
13932
- if (receipt.accessKey !== accessKey || !isRecord(observed) ||
13933
- typeof observed.epoch !== "string" || !Number.isFinite(observed.observedAt)) continue;
13934
- const reset = receipt.epoch !== "unknown" && observed.epoch !== receipt.epoch;
13935
- if (!reset && observed.observedAt < receipt.startedAt) continue;
13936
- changed = removeAcquisitionReceipt(state, id) || changed;
15048
+ const activeKeys = new Set(Object.values(state.subscriptions)
15049
+ .map((subscription) => state.queries[subscription.queryKey]?.query.accessKey)
15050
+ .filter(Boolean));
15051
+ const currentAccessKey = /^[0-9a-f]{64}$/.test(scope.accessKey ?? "")
15052
+ ? scope.accessKey : null;
15053
+ if (currentAccessKey) {
15054
+ for (const [hash, projection] of Object.entries(state.quotaProjections)) {
15055
+ if (hash !== scope.hash) projection.accessKeys = projection.accessKeys.filter((key) =>
15056
+ key !== currentAccessKey);
15057
+ }
13937
15058
  }
13938
- return { changed, value: { ...state.metrics } };
15059
+ const previous = state.quotaProjections[scope.hash];
15060
+ if (previous && previous.revision > ledger.revision) return { changed: false, value: false };
15061
+ if (!previous && Object.keys(state.quotaProjections).length >= ACQUISITION_MAX_QUOTA_PROJECTIONS) {
15062
+ const oldest = Object.entries(state.quotaProjections)
15063
+ .filter(([, projection]) => !projection.accessKeys.some((key) => activeKeys.has(key)))
15064
+ .sort((a, b) => a[1].at - b[1].at)[0];
15065
+ if (!oldest) return { changed: false, value: false };
15066
+ delete state.quotaProjections[oldest[0]];
15067
+ }
15068
+ const accessKeys = previous?.accessKeys.filter((key) => activeKeys.has(key)) ?? [];
15069
+ if (currentAccessKey && !accessKeys.includes(currentAccessKey) &&
15070
+ accessKeys.length < ACQUISITION_MAX_SUBSCRIPTIONS) accessKeys.push(currentAccessKey);
15071
+ state.quotaProjections[scope.hash] = {
15072
+ revision: ledger.revision, units, at: now(), status: "current", accessKeys,
15073
+ };
15074
+ return { value: true };
13939
15075
  });
13940
15076
  }
13941
15077
 
@@ -13948,7 +15084,7 @@ function createAcquisitionEngine({
13948
15084
  inspect,
13949
15085
  diagnostics,
13950
15086
  recordMetrics,
13951
- reconcileUncertainty,
15087
+ syncQuotaProjection,
13952
15088
  setHold,
13953
15089
  close,
13954
15090
  path,
@@ -13964,6 +15100,27 @@ function setRuntimeAcquisitionHold(engine, subscriptions, resource, reason, acce
13964
15100
  return engine.setHold(subscription.id, reason, { resource, accessKey });
13965
15101
  }
13966
15102
 
15103
+ function clearScheduledRuntimeControlHold(engine, subscriptions, resource, accessKey, observedAt) {
15104
+ const subscription = subscriptions?.get(resource);
15105
+ if (!subscription?.id) return { ok: false, reason: "stale" };
15106
+ if (typeof accessKey !== "string" || !Number.isFinite(observedAt)) {
15107
+ return { ok: false, reason: "stale-access" };
15108
+ }
15109
+ const inspected = engine.inspect(subscription.id);
15110
+ if (!inspected.ok) return inspected;
15111
+ const record = inspected.value;
15112
+ if (record?.query?.resource !== resource || record.query.accessKey !== accessKey) {
15113
+ return { ok: false, reason: "stale-access" };
15114
+ }
15115
+ const hold = record.hold;
15116
+ if (!hold) return { ok: true, value: "already-clear" };
15117
+ if (hold.reason !== "coordination") return { ok: true, value: "retained" };
15118
+ if (hold.at >= observedAt) return { ok: false, reason: "newer-hold" };
15119
+ const cleared = engine.setHold(subscription.id, null,
15120
+ { resource, accessKey, currentHold: hold });
15121
+ return cleared.ok ? { ok: true, value: "cleared" } : cleared;
15122
+ }
15123
+
13967
15124
  // The one place the cadence table is decided. It used to be spread across a
13968
15125
  // fixed active floor, a fixed four-floor background slot, and a Security-only
13969
15126
  // unchanged rule -- three policies that could disagree, and did: Security
@@ -13977,7 +15134,10 @@ function pollPolicyInterval({
13977
15134
  unchangedCount = 0,
13978
15135
  inProgressCI = false,
13979
15136
  background = "all",
15137
+ requested = false,
13980
15138
  }) {
15139
+ if (background === "on-demand" &&
15140
+ (demand !== "active" || (tab !== "actions" && !requested))) return Number.POSITIVE_INFINITY;
13981
15141
  if (demand === "none") return Number.POSITIVE_INFINITY;
13982
15142
  if (demand === "inactive") {
13983
15143
  if (background === "off") return Number.POSITIVE_INFINITY;
@@ -14105,6 +15265,7 @@ function pollSchedule({
14105
15265
  demand: isActive ? "active" : "inactive",
14106
15266
  unchangedCount: states[key]?.unchangedCount ?? 0,
14107
15267
  inProgressCI: states[key]?.inProgressCI === true,
15268
+ requested: states[key]?.requested === true,
14108
15269
  background,
14109
15270
  });
14110
15271
  if (!Number.isFinite(step)) {
@@ -14249,7 +15410,7 @@ function governorDataReady(refreshResult, snapshot, activeKey, nowMs) {
14249
15410
  return RATE_RESOURCES.every((resource) => {
14250
15411
  if (costs[resource] <= 0) return true;
14251
15412
  const budget = snapshot.value.budgets[resource];
14252
- const available = availableForGrant({ budget, resource, nowMs });
15413
+ const available = governorAvailableForGrant(snapshot.value, resource, nowMs);
14253
15414
  return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource) &&
14254
15415
  budget.blockUntil <= nowMs && available.mode === "open" &&
14255
15416
  available.spendable >= costs[resource];
@@ -14456,6 +15617,335 @@ function coordinationNotice(reason) {
14456
15617
  return "Can't coordinate API use — retrying";
14457
15618
  }
14458
15619
 
15620
+ // Recovery diagnostics have no request payloads, credential identifiers or
15621
+ // owner capabilities. The journal is advisory: a failed write must never deny
15622
+ // an otherwise valid data operation. The quota ledger remains authoritative.
15623
+ const RECOVERY_JOURNAL_MAX_ENTRIES = 128;
15624
+ const RECOVERY_JOURNAL_MAX_BYTES = 128 * 1024;
15625
+ const RECOVERY_ENTRY_MAX_BYTES = 1024;
15626
+ const RECOVERY_COPY = Object.freeze({
15627
+ busy: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
15628
+ stale: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
15629
+ "identity-busy": ["GitHub budget check is shared", "Retry automatically after the current check", "Budget check; retry"],
15630
+ "transport-busy": ["Shared GitHub request is in progress", "Retry automatically", "Request; retry"],
15631
+ unwritable: ["Local coordination cannot be written", "Run gh-glance --doctor; check storage permissions", "Store blocked; doctor"],
15632
+ corrupt: ["Local coordination is corrupt", "Run gh-glance --doctor; preserve the reported state file", "State corrupt; doctor"],
15633
+ capacity: ["Local coordination is full", "Run gh-glance --doctor; preserve the reported state file", "State full; doctor"],
15634
+ "accounting-overflow": ["Local coordination accounting overflow", "Run gh-glance --doctor; preserve the reported state file", "Overflow; doctor"],
15635
+ "reservations-invalid": ["Local coordination reservation invalid", "Run gh-glance --doctor; preserve the reported state file", "Reservation; doctor"],
15636
+ "external-factor-invalid": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
15637
+ "pacing-invalid": ["Local coordination pacing invalid", "Run gh-glance --doctor; preserve the reported state file", "Pacing invalid; doctor"],
15638
+ "budget-resource": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
15639
+ "legacy-unresolved": ["Older requests still reserve quota", "Verify old children ended; a verified system restart can resolve unknown ownership", "Old debt; restart"],
15640
+ "legacy-corrupt": ["Older coordination evidence is corrupt", "Run gh-glance --doctor; preserve the reported state file", "Old state; doctor"],
15641
+ "migration-hold": ["Older session still uses coordination", "Wait for the older quota reset; retry automatically at the shown time", "Old quota; wait"],
15642
+ "restart-required": ["Older session still uses coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
15643
+ "identity-backoff": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
15644
+ "identity-capacity": ["GitHub budget check allowance is full", "Retry automatically when the rolling window renews", "Budget check; retry"],
15645
+ "budget-unknown": ["GitHub budget check pending", "Retry automatically", "Budget check; retry"],
15646
+ "budget-stale": ["GitHub budget evidence is old", "Retry the shared budget check automatically", "Budget old; retry"],
15647
+ "budget-future": ["Clock changed; budget needs recheck", "Retry the shared budget check automatically", "Clock; retry"],
15648
+ "probe-failed": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
15649
+ "block-unpublished": ["Saving GitHub request limit", "Retry sharing the hold automatically", "Saving hold; retry"],
15650
+ "budget-reset": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
15651
+ reset: ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
15652
+ priority: ["Background refresh waiting its turn", "Retry automatically at the next slot", "Queued; retry"],
15653
+ "rate-limit": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
15654
+ "rate-limited": ["GitHub request limit", "Retry automatically after the shared hold", "Limit; retry"],
15655
+ "local-reserve": ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
15656
+ reserve: ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
15657
+ blocked: ["GitHub request hold", "Retry automatically when the hold ends", "Request hold; retry"],
15658
+ secondary: ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
15659
+ "secondary-rate-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
15660
+ "abuse-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
15661
+ "throttle-paused": ["Repeated GitHub request limit", "Press r to retry after checking the hold", "Limit; press r"],
15662
+ "network-outage": ["GitHub connection failed", "Retry automatically when the connection returns", "Network; retry"],
15663
+ "credential-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
15664
+ "identity-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
15665
+ disconnected: ["Collector connection unavailable", "Restore the selected collector connection", "Collector; connection"],
15666
+ "app-auth": ["GitHub App authorization unavailable", "Restore the selected App authorization", "App; restore auth"],
15667
+ "security-incomplete": ["Security data incomplete", "Run gh-glance --doctor --probe for endpoint access", "Security; doctor"],
15668
+ "receipt-retry": ["Saving request result", "Retry automatically without repeating the request", "Saving; retry"],
15669
+ "interrupted-request": ["Recovering interrupted request", "Retry source observation when quota allows", "Interrupted; retry"],
15670
+ "request-queue": ["Recovering local request queue", "Retry automatically after compaction", "Queue; retry"],
15671
+ "acquisition-busy": ["Acquisition store is busy", "Check the shared lock; retry automatically", "Store lock; retry"],
15672
+ "clock-recovery": ["Rechecking after sleep or clock change", "Retry source observation automatically", "Clock; retry"],
15673
+ "auth-problem": ["GitHub authorization failed", "Run gh auth status, then restore authorization", "Auth; gh auth status"],
15674
+ unavailable: ["Repository or endpoint unavailable", "Check the selected repository and permissions", "Repo; check access"],
15675
+ "source-incomplete": ["GitHub response incomplete", "Retry source observation automatically", "Incomplete; retry"],
15676
+ "unknown-scope": ["GitHub account scope unavailable", "Run gh auth status; retry when identity is verified", "Login; gh auth status"],
15677
+ "version-mismatch": ["Older session cannot share coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
15678
+ "unsafe-permissions": ["Local coordination permissions are unsafe", "Run gh-glance --doctor; repair directory permissions", "Permissions; doctor"],
15679
+ "disk-full": ["Local coordination storage is full", "Run gh-glance --doctor; free local storage", "Disk full; doctor"],
15680
+ coordination: ["Local coordination unavailable", "Run gh-glance --doctor; retry when storage works", "Coordination; doctor"],
15681
+ });
15682
+
15683
+ function recoveryCause({ reason, resource = null, at = Date.now(), retryAt = null,
15684
+ sourceAt = null, receiptUnits = null, debtUnits = null, oldestUnresolvedAt = null,
15685
+ observerAt = null, terminalizationBacklog = null, lastFailedTransition = null,
15686
+ origin = "coordination", tab = null } = {}) {
15687
+ const code = typeof reason === "string" && Object.hasOwn(RECOVERY_COPY, reason)
15688
+ ? reason : "coordination";
15689
+ const copy = RECOVERY_COPY[code] ?? RECOVERY_COPY.coordination;
15690
+ const time = (value) => Number.isFinite(value) && value >= 0 ? value : null;
15691
+ const units = (value) => Number.isSafeInteger(value) && value >= 0 ? value : null;
15692
+ return {
15693
+ code, resource: RATE_RESOURCES.includes(resource) ? resource : null,
15694
+ origin: origin === "source" ? "source" : "coordination",
15695
+ tab: TAB_KEYS.includes(tab) ? tab : null,
15696
+ at: time(at) ?? Date.now(), firstAt: time(at) ?? Date.now(),
15697
+ retryAt: time(retryAt), sourceAt: time(sourceAt),
15698
+ receiptUnits: units(receiptUnits), debtUnits: units(debtUnits),
15699
+ oldestUnresolvedAt: time(oldestUnresolvedAt), observerAt: time(observerAt),
15700
+ terminalizationBacklog: units(terminalizationBacklog),
15701
+ lastFailedTransition: typeof lastFailedTransition === "string" && Object.hasOwn(RECOVERY_COPY, lastFailedTransition)
15702
+ ? lastFailedTransition : code,
15703
+ summary: copy[0], action: copy[1], short: copy[2],
15704
+ };
15705
+ }
15706
+
15707
+ function shouldClearRecoveryCause(cause, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
15708
+ if (!cause) return false;
15709
+ if (scheduledHold !== null) {
15710
+ return scheduledHold.ok === true && scheduledHold.value === "cleared" &&
15711
+ cause.origin === "coordination" && ["busy", "stale", "acquisition-busy"].includes(cause.code);
15712
+ }
15713
+ return cause.origin !== "source" || sourceSuccess && cause.tab === tab;
15714
+ }
15715
+
15716
+ function recoveryRetryAt(decision, budget, observer, nowMs) {
15717
+ if (budget?.blockUntil > nowMs) return budget.blockUntil;
15718
+ if (["reservations-invalid", "external-factor-invalid", "pacing-invalid", "budget-resource",
15719
+ "accounting-overflow", "corrupt", "unwritable", "capacity"].includes(decision?.reason)) return null;
15720
+ const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision?.reason);
15721
+ if (decision?.mode === "probe" && !resetHold) {
15722
+ return observer?.nextAt > nowMs ? observer.nextAt : null;
15723
+ }
15724
+ const deadline = decision?.retryAt ?? decision?.resetMs ?? budget?.resetMs;
15725
+ return deadline > nowMs ? deadline : null;
15726
+ }
15727
+
15728
+ function terminalizationRecoveryCode(terminal) {
15729
+ if (terminal?.status === "retryable") return "receipt-retry";
15730
+ if (terminal?.status === "compacted") return "interrupted-request";
15731
+ if (terminal?.reason === "completion-capacity") return "capacity";
15732
+ return terminal?.reason ?? terminal?.status ?? "coordination";
15733
+ }
15734
+
15735
+ function requestFailureRecoveryCode(error, verdict) {
15736
+ if (verdict === "rate-limited") return "rate-limited";
15737
+ if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) return error.coordinationReason;
15738
+ if (verdict === "other" && error?.httpStarted === true) return "interrupted-request";
15739
+ return verdict === "other" ? "network-outage" : verdict;
15740
+ }
15741
+
15742
+ function requestFailureRecovery(error, verdict) {
15743
+ const reason = requestFailureRecoveryCode(error, verdict);
15744
+ return { reason, origin: reason === "transport-busy" ? "coordination" : "source" };
15745
+ }
15746
+
15747
+ function wallClockRecovery(previousAt, at, intervalMs) {
15748
+ if (!Number.isFinite(previousAt) || !Number.isFinite(at)) return null;
15749
+ return at < previousAt || at - previousAt > Math.max(30_000, 3 * intervalMs)
15750
+ ? "clock-recovery" : null;
15751
+ }
15752
+
15753
+ function collectorSourceAdvanced(previousAt, sourceAt) {
15754
+ return Number.isFinite(previousAt) && Number.isFinite(sourceAt) && sourceAt > previousAt;
15755
+ }
15756
+
15757
+ function cleanupQueueForResource(queue, resource) {
15758
+ let count = 0;
15759
+ for (const item of queue) if (tabRequestCost(item.cleanupKey)?.[resource] > 0) count += 1;
15760
+ return count;
15761
+ }
15762
+
15763
+ function presentRecovery(cause, { nowMs = Date.now(), cols = 80 } = {}) {
15764
+ if (!cause) return [];
15765
+ const width = Math.max(1, Number.isSafeInteger(cols) ? cols - 1 : 79);
15766
+ const resource = cause.resource === "graphql" ? "GraphQL" : cause.resource === "core" ? "Core" : "GitHub";
15767
+ const age = Number.isFinite(cause.sourceAt) && cause.sourceAt <= nowMs
15768
+ ? `cached ${formatDuration(nowMs - cause.sourceAt)}` : "cached age unavailable";
15769
+ const retry = Number.isFinite(cause.retryAt)
15770
+ ? `; next ${statusInterval(cause.retryAt, nowMs)}` : "";
15771
+ const summary = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
15772
+ ? `Older requests still reserve ${cause.debtUnits} units` : cause.summary;
15773
+ if (width < 60) {
15774
+ const action = cause.action.toLowerCase();
15775
+ const narrowAction = cause.code === "legacy-unresolved" ? "Check children; restart"
15776
+ : action.includes("gh-glance --doctor") ? "Run gh-glance --doctor"
15777
+ : action.includes("gh auth") ? "Run gh auth status"
15778
+ : action.includes("selected repository") ? "Check repository access"
15779
+ : action.includes("connection") ? "Restore connection"
15780
+ : action.includes("app authorization") ? "Restore App auth"
15781
+ : action.includes("permissions") ? "Check permissions"
15782
+ : action.includes("restart") ? "Restart and recheck"
15783
+ : action.includes("press r") ? "Press r to retry"
15784
+ : "Retry automatically";
15785
+ const narrowDeadline = Number.isFinite(cause.retryAt)
15786
+ ? statusInterval(cause.retryAt, nowMs) : null;
15787
+ const timedAction = narrowDeadline && cause.code === "migration-hold"
15788
+ ? `Wait; next ${narrowDeadline}`
15789
+ : narrowDeadline && narrowAction === "Retry automatically"
15790
+ ? `Retry; next ${narrowDeadline}`
15791
+ : narrowDeadline && narrowAction === "Press r to retry"
15792
+ ? `Press r after ${narrowDeadline}` : narrowAction;
15793
+ const short = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
15794
+ ? `Debt ${cause.debtUnits} units` : cause.short.split(";")[0];
15795
+ return [`${resource}: ${short}`.slice(0, width), timedAction.slice(0, width), age.slice(0, width)];
15796
+ }
15797
+ const full = `${resource}: ${summary}; ${cause.action}${retry}; ${age}`;
15798
+ if (full.length <= width) return [full];
15799
+ return [`${resource}: ${summary}`.slice(0, width),
15800
+ `${cause.action}${retry}`.slice(0, width), age.slice(0, width)];
15801
+ }
15802
+
15803
+ function recoveryJournalPath(scope) {
15804
+ return `${scope.path}.recovery.json`;
15805
+ }
15806
+
15807
+ function emptyRecoveryJournal() {
15808
+ return { version: 1, current: { core: null, graphql: null }, entries: [] };
15809
+ }
15810
+
15811
+ const RECOVERY_EVENT_KEYS = ["code", "resource", "at", "firstAt", "retryAt", "sourceAt",
15812
+ "receiptUnits", "debtUnits", "oldestUnresolvedAt", "observerAt", "terminalizationBacklog",
15813
+ "lastFailedTransition", "origin", "tab"];
15814
+
15815
+ function recoveryEvent(entry) {
15816
+ if (!isRecord(entry)) return null;
15817
+ if (exactKeys(entry, ["code", "resource", "at"]) && entry.code === "clear" &&
15818
+ RATE_RESOURCES.includes(entry.resource) && Number.isFinite(entry.at)) return entry;
15819
+ // An earlier clock sample can arrive after a later one for the same cause.
15820
+ // Repair only this bounded timestamp skew; every other field stays subject
15821
+ // to the strict journal validation below.
15822
+ const event = Number.isFinite(entry.at) && Number.isFinite(entry.firstAt) && entry.firstAt > entry.at
15823
+ ? { ...entry, firstAt: entry.at } : entry;
15824
+ if (!exactKeys(event, RECOVERY_EVENT_KEYS) || !RATE_RESOURCES.includes(event.resource) ||
15825
+ typeof event.code !== "string" || !Object.hasOwn(RECOVERY_COPY, event.code) ||
15826
+ !Number.isFinite(event.at) || !Number.isFinite(event.firstAt) ||
15827
+ ["retryAt", "sourceAt", "oldestUnresolvedAt", "observerAt"].some((key) =>
15828
+ event[key] !== null && (!Number.isFinite(event[key]) || event[key] < 0)) ||
15829
+ ["receiptUnits", "debtUnits", "terminalizationBacklog"].some((key) =>
15830
+ event[key] !== null && (!Number.isSafeInteger(event[key]) || event[key] < 0)) ||
15831
+ typeof event.lastFailedTransition !== "string" ||
15832
+ !Object.hasOwn(RECOVERY_COPY, event.lastFailedTransition) ||
15833
+ !["source", "coordination"].includes(event.origin) ||
15834
+ (event.tab !== null && !TAB_KEYS.includes(event.tab)) ||
15835
+ Buffer.byteLength(JSON.stringify(event)) > RECOVERY_ENTRY_MAX_BYTES) return null;
15836
+ return event;
15837
+ }
15838
+
15839
+ function visibleRecoveryEvent(entry) {
15840
+ if (!entry || entry.code === "clear") return entry;
15841
+ const copy = RECOVERY_COPY[entry.code] ?? RECOVERY_COPY.coordination;
15842
+ return { ...entry, summary: copy[0], action: copy[1], short: copy[2] };
15843
+ }
15844
+
15845
+ function readRecoveryTransitions(scope) {
15846
+ const path = recoveryJournalPath(scope);
15847
+ let raw;
15848
+ try {
15849
+ const file = lstatSync(path);
15850
+ if (!file.isFile() || file.size > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
15851
+ raw = readFileSync(path, "utf8");
15852
+ if (Buffer.byteLength(raw) > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
15853
+ } catch (error) {
15854
+ return { ...emptyRecoveryJournal(), status: error?.code === "ENOENT" ? "missing" : "unavailable" };
15855
+ }
15856
+ try {
15857
+ const journal = JSON.parse(raw);
15858
+ const entries = journal.entries?.map(recoveryEvent);
15859
+ const current = journal.current && Object.fromEntries(RATE_RESOURCES.map((resource) => [
15860
+ resource, journal.current[resource] === null ? null : recoveryEvent(journal.current[resource]),
15861
+ ]));
15862
+ if (!exactKeys(journal, ["version", "current", "entries"]) || journal.version !== 1 ||
15863
+ !exactKeys(journal.current, RATE_RESOURCES) || !Array.isArray(journal.entries) ||
15864
+ journal.entries.length > RECOVERY_JOURNAL_MAX_ENTRIES ||
15865
+ entries.some((entry) => entry === null) ||
15866
+ RATE_RESOURCES.some((resource) => journal.current[resource] !== null &&
15867
+ (current[resource] === null || current[resource].code === "clear"))) {
15868
+ return { ...emptyRecoveryJournal(), status: "corrupt" };
15869
+ }
15870
+ return { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
15871
+ [resource, visibleRecoveryEvent(current[resource])])),
15872
+ entries: entries.map(visibleRecoveryEvent), status: "healthy" };
15873
+ } catch {
15874
+ return { ...emptyRecoveryJournal(), status: "corrupt" };
15875
+ }
15876
+ }
15877
+
15878
+ function recordRecoveryTransition(scope, event) {
15879
+ if (!scope?.path || !RATE_RESOURCES.includes(event?.resource)) return { ok: false, reason: "invalid" };
15880
+ const path = recoveryJournalPath(scope);
15881
+ return withPersistenceLock(path, () => {
15882
+ const previous = readRecoveryTransitions(scope);
15883
+ if (!["healthy", "missing"].includes(previous.status)) return { ok: false, reason: previous.status };
15884
+ const plain = (entry) => entry && Object.fromEntries(RECOVERY_EVENT_KEYS.map((key) => [key, entry[key]]));
15885
+ const journal = { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
15886
+ [resource, plain(previous.current[resource])])),
15887
+ entries: previous.entries.map((entry) => entry.code === "clear" ? entry : plain(entry)) };
15888
+ const former = journal.current[event.resource];
15889
+ const at = Number.isFinite(event.at) ? event.at : Date.now();
15890
+ if (event.clear === true) {
15891
+ if (!former) return { ok: true, changed: false };
15892
+ journal.current[event.resource] = null;
15893
+ journal.entries.push({ code: "clear", resource: event.resource, at });
15894
+ } else {
15895
+ const entry = plain(recoveryCause({ ...event, reason: event.reason ?? event.code, at }));
15896
+ entry.firstAt = former?.code === entry.code && former?.origin === entry.origin &&
15897
+ former?.tab === entry.tab ? Math.min(former.firstAt, at) : at;
15898
+ if (Buffer.byteLength(JSON.stringify(entry)) > RECOVERY_ENTRY_MAX_BYTES) return { ok: false, reason: "capacity" };
15899
+ journal.current[event.resource] = entry;
15900
+ if (former?.code !== entry.code || former?.origin !== entry.origin || former?.tab !== entry.tab) {
15901
+ journal.entries.push(entry);
15902
+ }
15903
+ }
15904
+ journal.entries = journal.entries.slice(-RECOVERY_JOURNAL_MAX_ENTRIES);
15905
+ while (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES && journal.entries.length > 0) {
15906
+ journal.entries.shift();
15907
+ }
15908
+ if (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES) return { ok: false, reason: "capacity" };
15909
+ const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
15910
+ try {
15911
+ writeFileSync(temp, `${JSON.stringify(journal)}\n`, { mode: 0o600, flag: "wx" });
15912
+ renameSync(temp, path);
15913
+ return { ok: true, changed: true };
15914
+ } catch (error) {
15915
+ try { unlinkSync(temp); } catch { /* only our temporary file */ }
15916
+ return { ok: false, reason: error?.code ?? "unwritable" };
15917
+ }
15918
+ });
15919
+ }
15920
+
15921
+ function safeRecordRecoveryTransition(scope, event, write = recordRecoveryTransition) {
15922
+ try { return write(scope, event); } catch { return { ok: false, reason: "unavailable" }; }
15923
+ }
15924
+
15925
+ function recoveryLedgerSummary(scope, resource, nowMs = Date.now()) {
15926
+ if (!scope?.path || !RATE_RESOURCES.includes(resource)) return null;
15927
+ const loaded = readGovernorState(scope.path, nowMs, { persistMigration: false });
15928
+ if (!loaded.ok) return null;
15929
+ const state = loaded.value;
15930
+ const debt = state.debt[resource];
15931
+ const debtUnits = debt.unresolvedUnits + debt.quiescentUnits;
15932
+ const charged = governorChargedCost(state, resource, nowMs);
15933
+ const oldest = Object.values(state.debtGroups)
15934
+ .filter((group) => !group.quiescent && group.units[resource] > 0)
15935
+ .map((group) => group.at);
15936
+ return {
15937
+ receiptUnits: Number.isSafeInteger(charged) ? charged - debtUnits : null,
15938
+ debtUnits,
15939
+ oldestUnresolvedAt: oldest.length > 0 ? Math.min(...oldest) : null,
15940
+ observerAt: state.observers[resource]?.at > 0 && state.observers[resource].at <= nowMs
15941
+ ? state.observers[resource].at : null,
15942
+ debtCount: debt.unresolvedCount + debt.quiescentCount,
15943
+ receiptCount: Object.values(state.reservations).filter((reservation) =>
15944
+ reservationCost(reservation, resource, state.leases, nowMs) > 0).length +
15945
+ (reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs) > 0 ? 1 : 0),
15946
+ };
15947
+ }
15948
+
14459
15949
  function acquisitionFailureCopy(reason) {
14460
15950
  if (reason === "busy") return { short: "lock busy", narrow: "Shared lock busy", full: "Acquisition store is busy; checking the shared lock" };
14461
15951
  if (reason === "unwritable") return { short: "store blocked", narrow: "Store blocked", full: "Acquisition store cannot be written; check config permissions" };
@@ -14557,25 +16047,6 @@ function measuredNestedOperationCost(operation, value) {
14557
16047
  return actual;
14558
16048
  }
14559
16049
 
14560
- function uncertainReservationReceipts(scope, reservationId, nowMs) {
14561
- const snapshot = inspectGovernor(scope, nowMs);
14562
- const reservation = snapshot.ok ? snapshot.value.reservations?.[reservationId] : null;
14563
- const accessKey = scope.accessKey ?? scope.hash;
14564
- if (!reservation || typeof accessKey !== "string") return [];
14565
- return RATE_RESOURCES.flatMap((resource) => {
14566
- if (reservation.costs[resource] <= 0) return [];
14567
- return [{
14568
- id: `${reservationId}:${resource}`,
14569
- reservationId,
14570
- accessKey,
14571
- resource,
14572
- epoch: reservation.epochs[resource],
14573
- units: reservation.costs[resource],
14574
- startedAt: reservation.startedAt,
14575
- }];
14576
- });
14577
- }
14578
-
14579
16050
  function abortableDelay(ms, signal, {
14580
16051
  setTimeout: setTimeout_ = setTimeout,
14581
16052
  clearTimeout: clearTimeout_ = clearTimeout,
@@ -14770,30 +16241,38 @@ async function runAdmittedOperation({
14770
16241
  }
14771
16242
  const reservationId = admitted.value.reservationId;
14772
16243
  const settlementScope = { ...scope, identityProvider: null };
14773
- const pendingReceipts = uncertainReservationReceipts(scope, reservationId, now());
16244
+ const transportScope = { ...scope, receiptCapability: admitted.value.receiptCapability };
14774
16245
  try {
14775
- const value = await requestIdentityStorage.run(scope, () => run(signal));
16246
+ const value = await requestIdentityStorage.run(transportScope, () => run(signal));
14776
16247
  const measured = measuredNestedOperationCost(operation, value);
14777
16248
  const costs = measured ?? operationCost(operation);
14778
- const outcome = value?.ok === false ? governorOutcomeForError(value.failure) : "measured-success";
14779
- completeReservation(settlementScope, reservationId, {
16249
+ const outcome = value?.notStarted === true ? "measured-success" :
16250
+ value?.ok === false ? governorOutcomeForError(value.failure) : "measured-success";
16251
+ const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
14780
16252
  outcome,
14781
- ...(measured || outcome === "measured-success" ? { actualCost: costs } : {}),
14782
- }, now());
16253
+ ...(value?.notStarted === true ? { neverStarted: true } : {}),
16254
+ ...(measured || outcome === "measured-success" ? { actualCosts: costs } : {}),
16255
+ }, now);
16256
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
16257
+ return { ok: false, error: new Error(`API receipt settlement ${terminal.status}`),
16258
+ reservationId };
16259
+ }
14783
16260
  const identityCoordinator = scope.identityCoordinator ?? runtimeIdentityCoordinator;
14784
16261
  if (scope.accessKey && scope.accessKey !== identityCoordinator?.current()?.accessKey) {
14785
16262
  return { ok: false, error: new Error("Credential changed"), reservationId };
14786
16263
  }
14787
- return { ok: true, value, reservationId,
14788
- uncertainReceipts: measured ? [] : pendingReceipts };
16264
+ return { ok: true, value, reservationId };
14789
16265
  } catch (error) {
14790
16266
  const actualCost = measuredNestedOperationCost(operation, { requestMetrics: error?.requestMetrics });
14791
- const uncertainReceipts = actualCost ? [] : pendingReceipts;
14792
- completeReservation(settlementScope, reservationId, {
16267
+ const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
14793
16268
  outcome: governorOutcomeForError(error),
14794
- ...(actualCost ? { actualCost } : {}),
14795
- }, now());
14796
- return { ok: false, error, reservationId, uncertainReceipts };
16269
+ ...(error?.notStarted === true ? { neverStarted: true } : {}),
16270
+ ...(actualCost ? { actualCosts: actualCost } : {}),
16271
+ }, now);
16272
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
16273
+ error.receiptSettlement = terminal.status;
16274
+ }
16275
+ return { ok: false, error, reservationId };
14797
16276
  }
14798
16277
  }
14799
16278
 
@@ -15010,6 +16489,7 @@ function refreshStatus({
15010
16489
  stale = false,
15011
16490
  disconnected = false,
15012
16491
  screenReader = false,
16492
+ onDemand = false,
15013
16493
  } = {}) {
15014
16494
  const status = (kind, glyphKind, label, tone, animate = false, detailKind = null) => ({
15015
16495
  kind,
@@ -15046,6 +16526,7 @@ function refreshStatus({
15046
16526
  // Keep the cause visible until a successful result clears it.
15047
16527
  if (activeError) return status("failed", "failed", "Failed", "attention");
15048
16528
  if (securityIncomplete) return status("limited", "limited", "Limited", "attention");
16529
+ if (onDemand) return status("cached", "watching", "Cached", "inert");
15049
16530
  if (sharedData) return status("shared", "watching", "Shared", "inert", false, "sharing");
15050
16531
  if (["waiting", "pending", "probe"].includes(mode)) {
15051
16532
  const sharing = sharedLaneProvenance(governorDecision).waitCause === "shared-lane";
@@ -15535,6 +17016,8 @@ function App({ onCreateRemote = () => {} } = {}) {
15535
17016
  const [waiting, setWaiting] = useState({ actions: true, issues: true, prs: true, security: true });
15536
17017
  const [governorDecisions, setGovernorDecisions] = useState(probingGovernorDecisions);
15537
17018
  const [acquisitionFailures, setAcquisitionFailures] = useState({});
17019
+ const [recoveryCauses, setRecoveryCauses] = useState({ core: null, graphql: null });
17020
+ const recoveryRef = useRef(recoveryCauses);
15538
17021
  const [governorEpochs, setGovernorEpochs] = useState(null);
15539
17022
  const [requestStatuses, setRequestStatuses] = useState(() =>
15540
17023
  Object.fromEntries(TABS.map((candidate) => [candidate.key, null])),
@@ -15560,6 +17043,10 @@ function App({ onCreateRemote = () => {} } = {}) {
15560
17043
  ? { mode: "paused", reason: activeFailure, coordinationError: true,
15561
17044
  cause: acquisitionFailureCopy(activeFailure).short }
15562
17045
  : governorDecisions[tab.key];
17046
+ const activeResource = RATE_RESOURCES.find((resource) => tabRequestCost(tab.key)[resource] > 0) ?? null;
17047
+ const retainedRecovery = activeResource ? recoveryCauses[activeResource] : null;
17048
+ const activeRecovery = retainedRecovery?.origin === "source" && retainedRecovery.tab !== tab.key
17049
+ ? null : retainedRecovery;
15563
17050
  const activeRequestStatus = requestStatuses[tab.key];
15564
17051
  // Once any endpoint proves the folder has no remote, the whole dashboard is
15565
17052
  // in setup mode. Keeping this tab-local made a quick tab switch replace the
@@ -15590,7 +17077,10 @@ function App({ onCreateRemote = () => {} } = {}) {
15590
17077
  // Counts the lines actually rendered, not the notes collected -- getting this
15591
17078
  // wrong by one row is what makes ink repaint the whole frame.
15592
17079
  const extraLines =
15593
- NOTICE_ROWS +
17080
+ (activeRecovery || tabError || activeGovernorDecision?.coordinationError ||
17081
+ activeGovernorDecision?.mode === "paused" || activeGovernorDecision?.disconnected ||
17082
+ data[tab.key] !== null && !runtimeIdentityCoordinator?.current()
17083
+ ? 3 : NOTICE_ROWS) +
15594
17084
  (tab.key === "security" && !remoteSetup ? securityLines.length : 0);
15595
17085
  // Reserve lines for: the tab bar and the divider under it (2), the panel's
15596
17086
  // top and bottom edges (2), the column header and its separator (2), and
@@ -15641,6 +17131,14 @@ function App({ onCreateRemote = () => {} } = {}) {
15641
17131
  const ageMs = collectorCachedSourceAge(cachedTab);
15642
17132
  return ageMs === null ? [] : [[candidate.key, { ageMs, measuredAt: performance.now() }]];
15643
17133
  })));
17134
+ // Keep server source timestamps separate from client-local age estimates.
17135
+ // An SSH receipt's age is measured on the client and cannot prove a new
17136
+ // source observation when a cached snapshot is delivered again.
17137
+ const collectorSourceSuccessRef = useRef(Object.fromEntries(TABS.flatMap((candidate) => {
17138
+ const cachedTab = cachedEntry?.tabs[candidate.key];
17139
+ const sourceAt = cachedTab?.source?.lastSuccessAt ?? cachedTab?.lastOk;
17140
+ return Number.isFinite(sourceAt) ? [[candidate.key, sourceAt]] : [];
17141
+ })));
15644
17142
  const fetchTabRef = useRef(null);
15645
17143
  const contextCoordinatorRef = useRef(null);
15646
17144
  const governorRef = useRef(null);
@@ -16106,6 +17604,53 @@ function App({ onCreateRemote = () => {} } = {}) {
16106
17604
  : { ...current, [key]: value });
16107
17605
  }
16108
17606
 
17607
+ const scheduledControlAttempts = new Map();
17608
+
17609
+ function recordRuntimeRecovery(key, reason, { retryAt = null, origin = "coordination" } = {}) {
17610
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
17611
+ if (!resource) return;
17612
+ const at = Date.now();
17613
+ const previous = recoveryRef.current[resource];
17614
+ const candidate = recoveryCause({
17615
+ reason, resource, at, retryAt, sourceAt: lastOkRef.current[key] ?? null,
17616
+ origin, tab: origin === "source" ? key : null,
17617
+ });
17618
+ const hardFailure = ["corrupt", "unwritable", "capacity", "accounting-overflow"].includes(candidate.code);
17619
+ if (!hardFailure && previous?.code === "receipt-retry" &&
17620
+ terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
17621
+ if (!hardFailure && previous?.code === "request-queue" &&
17622
+ cleanupQueueForResource(cleanupQueue, resource) > 0) return;
17623
+ if (previous?.origin === "source" && previous.tab === key && origin === "coordination" &&
17624
+ !["corrupt", "unwritable", "capacity", "accounting-overflow",
17625
+ "receipt-retry", "interrupted-request"].includes(candidate.code)) return;
17626
+ if (previous?.code === candidate.code && previous?.retryAt === candidate.retryAt &&
17627
+ previous?.origin === candidate.origin && previous?.tab === candidate.tab &&
17628
+ at - previous.at < 60_000) return;
17629
+ const cause = recoveryCause({ ...candidate, reason: candidate.code,
17630
+ terminalizationBacklog: cleanupQueueForResource(cleanupQueue, resource) +
17631
+ terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource),
17632
+ ...recoveryLedgerSummary(scope, resource, at) });
17633
+ cause.firstAt = previous?.code === cause.code && previous?.origin === cause.origin &&
17634
+ previous?.tab === cause.tab ? previous.firstAt : at;
17635
+ recoveryRef.current = { ...recoveryRef.current, [resource]: cause };
17636
+ setRecoveryCauses(recoveryRef.current);
17637
+ if (scope) setTimeout(() => { safeRecordRecoveryTransition(scope, cause); }, 0).unref?.();
17638
+ }
17639
+
17640
+ function clearRuntimeRecovery(resource, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
17641
+ const previous = recoveryRef.current[resource];
17642
+ if (previous?.code === "receipt-retry" &&
17643
+ terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
17644
+ if (previous?.code === "request-queue" &&
17645
+ cleanupQueueForResource(cleanupQueue, resource) > 0) return;
17646
+ if (!shouldClearRecoveryCause(previous, { sourceSuccess, tab, scheduledHold })) return;
17647
+ recoveryRef.current = { ...recoveryRef.current, [resource]: null };
17648
+ setRecoveryCauses(recoveryRef.current);
17649
+ if (scope) setTimeout(() => {
17650
+ safeRecordRecoveryTransition(scope, { resource, clear: true, at: Date.now() });
17651
+ }, 0).unref?.();
17652
+ }
17653
+
16109
17654
  function publishGovernorEpochs(snapshot) {
16110
17655
  const epochs = snapshot?.value?.epochs;
16111
17656
  if (!isRecord(epochs)) return;
@@ -16135,7 +17680,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16135
17680
  return null;
16136
17681
  }
16137
17682
 
16138
- function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false) {
17683
+ function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false, retryAt = null) {
17684
+ recordRuntimeRecovery(key, acquisitionFailure && reason === "busy" ? "acquisition-busy" : reason,
17685
+ { retryAt });
16139
17686
  if (acquisitionFailure) setAcquisitionFailures((current) => current[key] === reason
16140
17687
  ? current : { ...current, [key]: reason ?? "unavailable" });
16141
17688
  const hold = diagnosticHold ?? (
@@ -16180,16 +17727,27 @@ function App({ onCreateRemote = () => {} } = {}) {
16180
17727
  return;
16181
17728
  }
16182
17729
  if (!snapshot?.ok) {
16183
- pauseCoordination(key, snapshot?.reason);
17730
+ pauseCoordination(key, snapshot?.reason, null, false, snapshot?.retryAt);
16184
17731
  return;
16185
17732
  }
16186
17733
  const tabReady = governorTabControlReady(snapshot, key, nowMs);
16187
17734
  if (!refreshed?.ok && !tabReady) {
16188
- pauseCoordination(key, refreshed?.reason, "observer");
17735
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0 &&
17736
+ snapshot.value.observers[candidate]?.outcome !== "healthy");
17737
+ pauseCoordination(key, refreshed?.reason, "observer", false,
17738
+ resource ? snapshot.value.observers[resource]?.nextAt : null);
16189
17739
  return;
16190
17740
  }
16191
17741
  publishGovernorEpochs(snapshot);
16192
17742
  if (refreshed.value?.status === "waiting" && !tabReady) {
17743
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
17744
+ if (resource && !recoveryRef.current[resource] && scope) {
17745
+ const shared = readRecoveryTransitions(scope).current[resource];
17746
+ if (shared?.origin === "coordination") {
17747
+ recoveryRef.current = { ...recoveryRef.current, [resource]: shared };
17748
+ setRecoveryCauses(recoveryRef.current);
17749
+ }
17750
+ }
16193
17751
  setTabGovernorDecision(key, { mode: "waiting", probing: true });
16194
17752
  return;
16195
17753
  }
@@ -16198,32 +17756,45 @@ function App({ onCreateRemote = () => {} } = {}) {
16198
17756
  const reservation = pendingIntent
16199
17757
  ? snapshot.value.reservations[`reservation:${pendingIntent.intentId}`]
16200
17758
  : null;
17759
+ const costs = tabRequestCost(key);
16201
17760
  if (reservation?.status === "scheduled" && reservation.notBefore > nowMs) {
17761
+ const accessKey = pendingIntent.cleanupScope?.accessKey;
17762
+ const reservationId = `reservation:${pendingIntent.intentId}`;
17763
+ const lastAttempt = scheduledControlAttempts.get(key);
17764
+ if (accessKey && pendingIntent.cleanupScope?.hash === scope?.hash &&
17765
+ identity()?.accessKey === accessKey &&
17766
+ (lastAttempt?.id !== reservationId || nowMs - lastAttempt.at >= 30_000)) {
17767
+ scheduledControlAttempts.set(key, { id: reservationId, at: nowMs });
17768
+ // A guarded hold write can retire an older transient notice. Merely
17769
+ // reading a future grant or an absent hold cannot prove recovery.
17770
+ const cleared = clearScheduledRuntimeControlHold(acquisitionEngine,
17771
+ acquisitionSubscriptions, key, accessKey, nowMs);
17772
+ for (const resource of RATE_RESOURCES) {
17773
+ if (costs[resource] > 0) clearRuntimeRecovery(resource, { scheduledHold: cleared });
17774
+ }
17775
+ }
16202
17776
  setTabGovernorDecision(key, visibleGovernorDecision({
16203
17777
  status: "scheduled",
16204
- reservationId: `reservation:${pendingIntent.intentId}`,
17778
+ reservationId,
16205
17779
  ...reservation,
16206
17780
  ...currentSharedLaneProvenance(pendingIntent, snapshot.value.leases, nowMs),
16207
17781
  }));
16208
17782
  return;
16209
17783
  }
16210
- const costs = tabRequestCost(key);
16211
17784
  for (const resource of RATE_RESOURCES) {
16212
17785
  if (costs[resource] <= 0) continue;
16213
17786
  const budget = snapshot.value.budgets[resource];
16214
17787
  const decision = budget
16215
- ? resourceDecision({
16216
- budget,
16217
- resource,
16218
- leases: snapshot.value.leases,
16219
- reservations: Object.values(snapshot.value.reservations),
16220
- nowMs,
16221
- cost: costs[resource],
16222
- })
17788
+ ? governorResourceDecision(snapshot.value, resource, nowMs, costs[resource])
16223
17789
  : { mode: "probe", reason: "budget-unknown" };
16224
17790
  if (decision.mode === "open") continue;
16225
17791
  const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision.reason) ||
16226
17792
  budget?.blockUntil > nowMs;
17793
+ const holdReason = budget?.blockUntil > nowMs
17794
+ ? budget.blockReason ?? decision.reason : decision.reason;
17795
+ recordRuntimeRecovery(key, holdReason, {
17796
+ retryAt: recoveryRetryAt(decision, budget, snapshot.value.observers[resource], nowMs),
17797
+ });
16227
17798
  setRuntimeAcquisitionHold(
16228
17799
  acquisitionEngine,
16229
17800
  acquisitionSubscriptions,
@@ -16235,7 +17806,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16235
17806
  );
16236
17807
  setTabGovernorDecision(key, {
16237
17808
  mode: resetHold || decision.mode === "paused" ? "paused" : "waiting",
16238
- reason: decision.reason,
17809
+ reason: holdReason,
16239
17810
  resetMs: budget?.resetMs ?? decision.resetMs ?? null,
16240
17811
  probing: decision.mode === "probe" && !resetHold,
16241
17812
  });
@@ -16249,6 +17820,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16249
17820
  identity()?.accessKey ?? null,
16250
17821
  );
16251
17822
  setTabGovernorDecision(key, null);
17823
+ for (const resource of RATE_RESOURCES) {
17824
+ if (costs[resource] > 0) clearRuntimeRecovery(resource);
17825
+ }
16252
17826
  }
16253
17827
 
16254
17828
  async function refreshRuntimeBudget(currentScope) {
@@ -16260,15 +17834,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16260
17834
  return refreshed;
16261
17835
  }
16262
17836
 
16263
- function reconcileRuntimeUncertainty(currentScope, snapshot) {
17837
+ function syncRuntimeQuotaProjection(currentScope, snapshot) {
16264
17838
  if (!snapshot?.ok) return;
16265
- const evidence = Object.fromEntries(RATE_RESOURCES.flatMap((resource) => {
16266
- const observer = snapshot.value.observers?.[resource];
16267
- const epoch = snapshot.value.epochs?.[resource];
16268
- return observer?.outcome === "healthy" && typeof epoch === "string" && Number.isFinite(observer.at)
16269
- ? [[resource, { epoch, observedAt: observer.at }]] : [];
16270
- }));
16271
- acquisitionEngine.reconcileUncertainty(currentScope.accessKey, evidence);
17839
+ acquisitionEngine.syncQuotaProjection(currentScope);
16272
17840
  }
16273
17841
 
16274
17842
  function commit(key, run, {
@@ -16320,7 +17888,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16320
17888
  : run;
16321
17889
  return requestIdentityStorage.run(scope, fencedRun)
16322
17890
  .then(async (result) => {
16323
- settleReservationWithBudgetObservations(settlementScope, leaseId, reservationId, result?.measuredSuccess === false
17891
+ const terminal = queueTerminalization(settlementScope, scope.receiptCapability, result?.measuredSuccess === false
16324
17892
  ? { outcome: "rejected", observations: result?.observations ?? [] }
16325
17893
  : {
16326
17894
  outcome: "measured-success",
@@ -16329,7 +17897,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16329
17897
  graphql: result?.graphqlSpent ?? GRAPHQL_PER_FETCH[key] ?? 0,
16330
17898
  },
16331
17899
  observations: result?.observations ?? [],
16332
- }, Date.now());
17900
+ });
17901
+ if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
17902
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
17903
+ failAcquisition(acquisition, { requestMetrics: acquisitionRequestMetrics(result), hold: "primary" });
17904
+ pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
17905
+ return;
17906
+ }
16333
17907
  if (!currentAccess()) {
16334
17908
  failAcquisition(acquisition, {
16335
17909
  requestMetrics: acquisitionRequestMetrics(result),
@@ -16352,7 +17926,6 @@ function App({ onCreateRemote = () => {} } = {}) {
16352
17926
  acquisition.capabilities = result.capabilities ?? {};
16353
17927
  acquisition.repositoryIdentity = result.repositoryIdentity;
16354
17928
  acquisition.requestMetrics = acquisitionRequestMetrics(result);
16355
- acquisition.uncertainReceipts = result.uncertainReceipts ?? [];
16356
17929
  }
16357
17930
  // Identical payload: skip the parse *and* the state update. Returning
16358
17931
  // the same state object makes React bail out of the re-render, so an
@@ -16387,9 +17960,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16387
17960
  // so the next admitted tick parses its response instead of taking the
16388
17961
  // identical-output fast path.
16389
17962
  if (transition.kind === "unusable") {
17963
+ recordRuntimeRecovery(key, "source-incomplete", { origin: "source" });
16390
17964
  failAcquisition(acquisition, {
16391
17965
  requestMetrics: acquisition?.requestMetrics ?? {},
16392
- uncertainReceipts: acquisition?.uncertainReceipts ?? [],
16393
17966
  hold: "primary",
16394
17967
  });
16395
17968
  rawRef.current[key] = transition.nextRaw;
@@ -16402,9 +17975,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16402
17975
  // freshness. Do not retain its raw value either, so the next source
16403
17976
  // retry is parsed instead of taking the identical-payload fast path.
16404
17977
  if (transition.kind === "blind") {
17978
+ recordRuntimeRecovery(key, "security-incomplete", { origin: "source" });
16405
17979
  failAcquisition(acquisition, {
16406
17980
  requestMetrics: acquisition?.requestMetrics ?? {},
16407
- uncertainReceipts: acquisition?.uncertainReceipts ?? [],
16408
17981
  hold: "primary",
16409
17982
  });
16410
17983
  rawRef.current[key] = transition.nextRaw;
@@ -16439,6 +18012,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16439
18012
  publishStagedEntities(entityRef.current, stagedEntities, transition.kind);
16440
18013
  if (transition.kind === "unchanged") {
16441
18014
  lastOkRef.current[key] = completedAt;
18015
+ for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
18016
+ clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
18017
+ }
16442
18018
  clearBackoff(`tab:${key}`);
16443
18019
  setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
16444
18020
  const cachedTab = pick(
@@ -16455,6 +18031,9 @@ function App({ onCreateRemote = () => {} } = {}) {
16455
18031
  const tabData = transition.data;
16456
18032
  const tabMeta = transition.meta;
16457
18033
  lastOkRef.current[key] = completedAt;
18034
+ for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
18035
+ clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
18036
+ }
16458
18037
  clearBackoff(`tab:${key}`);
16459
18038
  rawRef.current[key] = transition.nextRaw;
16460
18039
  setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
@@ -16474,10 +18053,14 @@ function App({ onCreateRemote = () => {} } = {}) {
16474
18053
  requestMetrics: err?.requestMetrics ?? {},
16475
18054
  hold: acquisitionFailureHold(err),
16476
18055
  });
16477
- settleReservationWithBudgetObservations(settlementScope, leaseId, reservationId, {
18056
+ const terminal = queueTerminalization(settlementScope, scope.receiptCapability, {
16478
18057
  outcome: governorOutcomeForError(err),
16479
18058
  observations: err?.budgetObservations ?? [],
16480
- }, Date.now());
18059
+ });
18060
+ if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
18061
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
18062
+ pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
18063
+ }
16481
18064
  if (cancelled || !currentAccess() || err?.name === "AbortError") return;
16482
18065
  // Preserve both the verdict and the bounded raw error in state. The
16483
18066
  // renderer translates recognized verdicts at draw time, which lets a
@@ -16494,6 +18077,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16494
18077
  // recover on the very next tick once the network is back.
16495
18078
  const steps = verdict === "rate-limited" ? null : pick(FAILURE_LADDER, verdict, null);
16496
18079
  if (steps) recordFailure(`tab:${key}`, performance.now(), steps);
18080
+ const backoffUntil = alertBackoff.get(backoffStorageKey(`tab:${key}`))?.until;
18081
+ const retryAt = Number.isFinite(backoffUntil)
18082
+ ? Date.now() + Math.max(0, backoffUntil - performance.now()) : null;
18083
+ if (terminal.status !== "blocked" && terminal.status !== "compacted") {
18084
+ const recovery = requestFailureRecovery(err, verdict);
18085
+ recordRuntimeRecovery(key, recovery.reason, { origin: recovery.origin, retryAt });
18086
+ }
16497
18087
  if (verdict === "rate-limited") {
16498
18088
  const snapshot = inspectGovernor(scope, Date.now());
16499
18089
  const costs = tabRequestCost(key);
@@ -16508,6 +18098,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16508
18098
  });
16509
18099
  }
16510
18100
  handleRateLimitBlocks(key, blocks, Date.now());
18101
+ const held = inspectGovernor(scope, Date.now());
18102
+ const heldUntil = RATE_RESOURCES.filter((resource) => costs[resource] > 0)
18103
+ .map((resource) => held.value?.budgets?.[resource]?.blockUntil)
18104
+ .filter((deadline) => Number.isFinite(deadline) && deadline > Date.now());
18105
+ if (heldUntil.length > 0) recordRuntimeRecovery(key, "rate-limited", {
18106
+ origin: "source", retryAt: Math.min(...heldUntil),
18107
+ });
16511
18108
  }
16512
18109
  })
16513
18110
  .finally(() => {
@@ -16541,6 +18138,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16541
18138
  const leaseId = governorId();
16542
18139
  const pending = new Map();
16543
18140
  const cleanupQueue = new Set();
18141
+ let lastWallAt = Date.now();
16544
18142
  // One manual handoff per tab, carrying which key produced it. A keypress
16545
18143
  // during an automatic request must not be lost, but repeated keypresses
16546
18144
  // during the manual request itself must not create a trailing second batch.
@@ -16579,12 +18177,20 @@ function App({ onCreateRemote = () => {} } = {}) {
16579
18177
  const unchangedPolls = Object.fromEntries(TAB_KEYS.map((key) => [key, 0]));
16580
18178
  let pendingBlockPublications = new Map();
16581
18179
  const acquisitionSubscriptions = new Map();
18180
+ // One-shot demand survives admission delays, but ends when an observation arrives.
18181
+ const requestedTabs = new Map();
16582
18182
 
16583
18183
  function adoptAcquisitionSnapshot(key, snapshot, receipt = null) {
16584
18184
  if (!snapshot || cancelled) return;
18185
+ if (requestedTabs.has(key) && !queuedManual.has(key) &&
18186
+ snapshot.lastSuccessAt >= requestedTabs.get(key)) {
18187
+ requestedTabs.delete(key);
18188
+ if (runtime.background === "on-demand" && key !== "actions") pollDueAt[key] = Infinity;
18189
+ }
16585
18190
  if (runtime.connect === null && Number.isFinite(snapshot.nextDueAt)) {
16586
18191
  sharedSnapshotDueAt = { ...sharedSnapshotDueAt, [key]: snapshot.nextDueAt };
16587
- if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key) {
18192
+ if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key &&
18193
+ Number.isFinite(pollIntervalFor(key, key))) {
16588
18194
  const at = Date.now();
16589
18195
  const due = sharedAcquisitionPollDeadline(pollDueAt[key], snapshot.nextDueAt, at, true);
16590
18196
  if (due < pollDueAt[key]) {
@@ -16657,6 +18263,14 @@ function App({ onCreateRemote = () => {} } = {}) {
16657
18263
  const host = effectiveRuntimeHost({ remoteUrls: runtimeRemoteUrls });
16658
18264
  const handles = new Map();
16659
18265
  const receivedSnapshots = new Set();
18266
+ const initialKey = TABS[activeIndexRef.current].key;
18267
+ const collectorDemand = (key) => ({
18268
+ active: TABS[activeIndexRef.current].key === key &&
18269
+ (runtime.background !== "on-demand" || key === "actions"),
18270
+ background: runtime.background === "all",
18271
+ floorMs: runtime.refreshMs,
18272
+ pages: pageStateRef.current[key]?.pages ?? 1,
18273
+ });
16660
18274
  let collectorReady = false;
16661
18275
  setGovernorDecisions(Object.fromEntries(TAB_KEYS.map((key) => [key, { disconnected: true }])));
16662
18276
  if (!repository || !host) {
@@ -16686,39 +18300,46 @@ function App({ onCreateRemote = () => {} } = {}) {
16686
18300
  },
16687
18301
  });
16688
18302
  for (const key of TAB_KEYS) {
16689
- const demand = {
16690
- active: TABS[activeIndexRef.current].key === key,
16691
- background: runtime.background !== "off",
16692
- floorMs: runtime.refreshMs,
16693
- pages: pageStateRef.current[key]?.pages ?? 1,
16694
- };
18303
+ const demand = collectorDemand(key);
16695
18304
  const subscribed = client.subscribe({ host, repo: repository, resource: key, demand,
16696
18305
  sourceCheckpoint: cachedEntry?.tabs[key]?.source ?? null,
16697
18306
  onSnapshot: (snapshot, receipt) => {
16698
18307
  receivedSnapshots.add(key);
18308
+ const sourceAdvanced = collectorSourceAdvanced(
18309
+ collectorSourceSuccessRef.current[key], snapshot.lastSuccessAt);
16699
18310
  adoptAcquisitionSnapshot(key, snapshot, receipt);
18311
+ if (Number.isFinite(snapshot.lastSuccessAt) &&
18312
+ (!Number.isFinite(collectorSourceSuccessRef.current[key]) ||
18313
+ snapshot.lastSuccessAt > collectorSourceSuccessRef.current[key])) {
18314
+ collectorSourceSuccessRef.current[key] = snapshot.lastSuccessAt;
18315
+ }
18316
+ if (sourceAdvanced) {
18317
+ for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
18318
+ clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
18319
+ }
18320
+ }
16700
18321
  setErrors((current) => current[key] === null ? current : { ...current, [key]: null });
16701
18322
  setWaiting((current) => current[key] ? { ...current, [key]: false } : current);
16702
18323
  setGovernorDecisions((current) => ({ ...current, [key]:
16703
18324
  collectorDisplayDecision({ connected: collectorReady, hasSnapshot: true }) }));
16704
18325
  },
16705
- onHold: (hold) => setGovernorDecisions((current) => ({ ...current, [key]:
16706
- collectorDisplayDecision({ connected: collectorReady,
16707
- hasSnapshot: receivedSnapshots.has(key), hold }) })),
18326
+ onHold: (hold) => {
18327
+ if (hold === "app-auth") recordRuntimeRecovery(key, "app-auth", { origin: "source" });
18328
+ setGovernorDecisions((current) => ({ ...current, [key]:
18329
+ collectorDisplayDecision({ connected: collectorReady,
18330
+ hasSnapshot: receivedSnapshots.has(key), hold }) }));
18331
+ },
16708
18332
  });
16709
18333
  if (subscribed.ok) handles.set(key, subscribed.value);
16710
18334
  }
18335
+ if (runtime.background === "on-demand" && initialKey !== "actions") {
18336
+ handles.get(initialKey)?.refresh();
18337
+ }
16711
18338
  fetchTabRef.current = (key, request = {}) => {
16712
18339
  const handle = handles.get(key);
16713
18340
  if (!handle) return;
16714
- if (request.kind === "tab-switch") {
16715
- for (const [candidate, candidateHandle] of handles) {
16716
- candidateHandle.updateDemand({ active: candidate === key, background: runtime.background !== "off",
16717
- floorMs: runtime.refreshMs, pages: pageStateRef.current[candidate]?.pages ?? 1 });
16718
- }
16719
- } else {
16720
- handle.updateDemand({ active: true, background: runtime.background !== "off",
16721
- floorMs: runtime.refreshMs, pages: pageStateRef.current[key]?.pages ?? 1 });
18341
+ for (const [candidate, candidateHandle] of handles) {
18342
+ candidateHandle.updateDemand(collectorDemand(candidate));
16722
18343
  }
16723
18344
  handle.refresh(request.force === true);
16724
18345
  };
@@ -16795,12 +18416,11 @@ function App({ onCreateRemote = () => {} } = {}) {
16795
18416
  demand: acquisition.demand?.active ? "active" : "inactive",
16796
18417
  unchangedCount: publicationView.unchangedCount,
16797
18418
  inProgressCI: key === "actions" && actionsInProgress(rows),
16798
- background: runtime.background,
18419
+ background: runtime.background === "on-demand" ? "all" : runtime.background,
16799
18420
  }),
16800
18421
  hold: null,
16801
18422
  capabilities: acquisition.capabilities ?? {},
16802
18423
  requestMetrics: acquisition.requestMetrics ?? {},
16803
- uncertainReceipts: acquisition.uncertainReceipts ?? [],
16804
18424
  repositoryIdentity: acquisition.repositoryIdentity,
16805
18425
  meta,
16806
18426
  securityNotes: key === "security"
@@ -16841,10 +18461,17 @@ function App({ onCreateRemote = () => {} } = {}) {
16841
18461
  automaticStatusVisible) {
16842
18462
  const descriptor = tabForKey(key);
16843
18463
  let acquisitionStartAttempted = false;
18464
+ let transportScope = null;
16844
18465
  const transported = await runStartedAcquisitionTransport(
16845
18466
  () => {
16846
18467
  const admitted = inspectGovernor(currentScope, nowMs);
16847
18468
  if (!admitted.ok) return admitted;
18469
+ const admittedReservation = admitted.value.reservations[reservationId];
18470
+ if (!admittedReservation?.receipt || admittedReservation.status !== "started") {
18471
+ return { ok: false, reason: "compacted" };
18472
+ }
18473
+ transportScope = { ...currentScope,
18474
+ receiptCapability: receiptCapabilityFromReservation(reservationId, admittedReservation) };
16848
18475
  acquisitionStartAttempted = true;
16849
18476
  const receipt = {
16850
18477
  reservationId,
@@ -16875,13 +18502,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16875
18502
  force: item.force,
16876
18503
  catalog: workflowCatalogRef.current,
16877
18504
  pages: item.acquisition.demand?.pages ?? pageStateRef.current[key]?.pages ?? 1,
16878
- governor: { scope: currentScope, leaseId },
18505
+ governor: { scope: transportScope, leaseId },
16879
18506
  previousRaw: rawRef.current[key] ?? null,
16880
18507
  }),
16881
18508
  {
16882
18509
  force: item.force,
16883
18510
  manual: item.manual === true,
16884
- scope: currentScope,
18511
+ scope: transportScope,
16885
18512
  reservationId,
16886
18513
  admittedAt: nowMs,
16887
18514
  automaticStatusVisible,
@@ -16892,6 +18519,13 @@ function App({ onCreateRemote = () => {} } = {}) {
16892
18519
  },
16893
18520
  );
16894
18521
  if (transported.ok) return transported;
18522
+ if (transportScope?.receiptCapability) {
18523
+ const terminal = queueTerminalization(currentScope,
18524
+ transportScope.receiptCapability, { outcome: "rejected", neverStarted: true });
18525
+ if (terminal.status === "blocked" || terminal.status === "compacted") {
18526
+ pauseCoordination(key, terminal.reason ?? terminal.status, null, true);
18527
+ }
18528
+ }
16895
18529
  pauseCoordination(key, transported.reason, null, acquisitionStartAttempted);
16896
18530
  // The governor reservation is already started. Even a transient failed
16897
18531
  // claim write cannot safely rebind it; retain its charge and get a fresh
@@ -16915,6 +18549,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16915
18549
  demand: key === activeKey ? "active" : "inactive",
16916
18550
  unchangedCount: unchangedPolls[key],
16917
18551
  inProgressCI: key === "actions" && actionsInProgress(actionRows),
18552
+ requested: requestedTabs.has(key),
16918
18553
  background: runtime.background,
16919
18554
  });
16920
18555
  }
@@ -16925,7 +18560,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16925
18560
  const localDueAt = Number.isFinite(step) ? at + step : Number.POSITIVE_INFINITY;
16926
18561
  pollDueAt = {
16927
18562
  ...pollDueAt,
16928
- [key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active),
18563
+ [key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active && Number.isFinite(step)),
16929
18564
  };
16930
18565
  }
16931
18566
 
@@ -16937,6 +18572,7 @@ function App({ onCreateRemote = () => {} } = {}) {
16937
18572
  function openPollDeadlines(at) {
16938
18573
  const activeKey = TABS[activeIndexRef.current].key;
16939
18574
  const opened = { ...NEVER_DUE };
18575
+ if (runtime.background === "on-demand") requestedTabs.set(activeKey, at);
16940
18576
  opened[activeKey] = at + 1;
16941
18577
  let slot = 1;
16942
18578
  for (const key of TAB_KEYS) {
@@ -17038,6 +18674,8 @@ function App({ onCreateRemote = () => {} } = {}) {
17038
18674
  coordinator.invalidate();
17039
18675
  setFailureContext(null);
17040
18676
  if (discardData) {
18677
+ recoveryRef.current = { core: null, graphql: null };
18678
+ setRecoveryCauses(recoveryRef.current);
17041
18679
  rawRef.current = {};
17042
18680
  entityRef.current.clear();
17043
18681
  lastOkRef.current = {};
@@ -17115,6 +18753,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17115
18753
  }
17116
18754
  scope = createQuotaScope(current, { root: runtimeIdentityCoordinator.root, identityProvider: identity });
17117
18755
  cleanupScope = { ...scope, identityProvider: null };
18756
+ const savedRecovery = readRecoveryTransitions(scope);
18757
+ recoveryRef.current = savedRecovery.status === "healthy"
18758
+ ? { ...savedRecovery.current } : { core: null, graphql: null };
18759
+ setRecoveryCauses(recoveryRef.current);
17118
18760
  const nextTarget = dashboardCacheTarget({ repo: runtime.repo, ghRepo: process.env.GH_REPO, host: current.host, account: current.accessKey });
17119
18761
  // The mount-time target had to guess the account, and guesses
17120
18762
  // "unverified" whenever the bounded warm resolution above did not land
@@ -17176,6 +18818,11 @@ function App({ onCreateRemote = () => {} } = {}) {
17176
18818
  function finishPendingCleanup(key, item) {
17177
18819
  if (pending.get(key) === item) pending.delete(key);
17178
18820
  cleanupQueue.delete(item);
18821
+ const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
18822
+ if (resource && cleanupQueueForResource(cleanupQueue, resource) === 0 &&
18823
+ recoveryRef.current[resource]?.code === "request-queue") {
18824
+ clearRuntimeRecovery(resource);
18825
+ }
17179
18826
  setAcquisitionFailures((current) => {
17180
18827
  if (!Object.hasOwn(current, key)) return current;
17181
18828
  const next = { ...current };
@@ -17204,7 +18851,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17204
18851
  }
17205
18852
 
17206
18853
  function retryPendingCleanup(key, item, at = Date.now()) {
17207
- if (item.cleanupInFlight) return false;
18854
+ if (item.cleanupInFlight) {
18855
+ recordRuntimeRecovery(key, "request-queue");
18856
+ return false;
18857
+ }
17208
18858
  item.cleanupPending = true;
17209
18859
  item.cleanupKey = key;
17210
18860
  cleanupQueue.add(item);
@@ -17230,6 +18880,7 @@ function App({ onCreateRemote = () => {} } = {}) {
17230
18880
  finishPendingCleanup(key, item);
17231
18881
  }
17232
18882
  if (!item.governorClean) armWake("data", governorControlRetryAt(at, runtime.refreshMs), dataWake);
18883
+ if (pending.has(key)) recordRuntimeRecovery(key, "request-queue");
17233
18884
  return !pending.has(key);
17234
18885
  }
17235
18886
 
@@ -17252,7 +18903,7 @@ function App({ onCreateRemote = () => {} } = {}) {
17252
18903
  // the tab's failure ladder. `force` is the separate, stronger request that
17253
18904
  // also drops validators -- the `R` key. `r` is manual and not forced, which
17254
18905
  // is what makes a quiet refresh cost nothing.
17255
- async function requestTab(key, kind = "active", { force = false } = {}) {
18906
+ async function requestTab(key, kind = "active", { force = false, detachTransport = false } = {}) {
17256
18907
  const manual = kind === "manual";
17257
18908
  const signal = controller.signal;
17258
18909
  const monotonicNow = performance.now();
@@ -17333,7 +18984,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17333
18984
  currentScope.accessKey,
17334
18985
  );
17335
18986
  if (ownership.value.role !== "producer") {
17336
- if (ownership.value.snapshot) adoptAcquisitionSnapshot(key, ownership.value.snapshot);
18987
+ if (ownership.value.snapshot) {
18988
+ if (ownership.value.reason === "fresh") requestedTabs.delete(key);
18989
+ adoptAcquisitionSnapshot(key, ownership.value.snapshot);
18990
+ }
17337
18991
  if (ownership.value.sharingCount > 1) {
17338
18992
  setTabGovernorDecision(key, {
17339
18993
  mode: "waiting",
@@ -17457,9 +19111,19 @@ function App({ onCreateRemote = () => {} } = {}) {
17457
19111
  setTabWaiting(key, false);
17458
19112
  const item = pending.get(key);
17459
19113
  if (!item) return { persisted: false, retry: false, kind, key };
17460
- await startPendingAcquisitionTransport(
19114
+ const transport = startPendingAcquisitionTransport(
17461
19115
  key, item, currentScope, decision.reservationId, nowMs, signal, false,
17462
19116
  );
19117
+ if (detachTransport) {
19118
+ // The scheduling single-flight owns admission, not the HTTP lifetime.
19119
+ // A background response must not block the next active-tab wake.
19120
+ void transport.catch((error) => {
19121
+ if (cancelled || pending.get(key) !== item) return;
19122
+ item.retryOnCleanup = true;
19123
+ cancelPending(key, currentScope);
19124
+ pauseCoordination(key, error?.reason ?? "stale");
19125
+ });
19126
+ } else await transport;
17463
19127
  return { persisted: true, retry: false, kind, key };
17464
19128
  }
17465
19129
  fetchTabRef.current = (key, { force = false, kind = force ? "manual" : "active" } = {}) => {
@@ -17470,6 +19134,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17470
19134
  unchangedPolls[key] = advanceUnchangedCount(unchangedPolls[key], "subscribed");
17471
19135
  }
17472
19136
  const requestedAt = Date.now();
19137
+ if (runtime.background === "on-demand") {
19138
+ if (kind === "tab-switch") requestedTabs.clear();
19139
+ requestedTabs.set(key, requestedAt);
19140
+ }
17473
19141
  // Manual work replaces the active poll that would otherwise be due. Move
17474
19142
  // that deadline before touching the governor so a data wake already in
17475
19143
  // this event-loop turn cannot start a second batch as soon as the forced
@@ -17583,11 +19251,10 @@ function App({ onCreateRemote = () => {} } = {}) {
17583
19251
  const budget = snapshot.value.budgets[resource];
17584
19252
  const observer = snapshot.value.observers[resource];
17585
19253
  const decision = budget
17586
- ? availableForGrant({ budget, resource, nowMs })
19254
+ ? governorAvailableForGrant(snapshot.value, resource, nowMs)
17587
19255
  : { mode: "paused" };
17588
- const held = pendingBlockPublications.has(resource) ||
17589
- budget?.blockUntil > nowMs ||
17590
- decision.mode === "paused" && decision.reason === "budget-exhausted";
19256
+ const held = governorResourceHeld(snapshot.value, resource, nowMs,
19257
+ pendingBlockPublications.has(resource));
17591
19258
  const retryAt = budget?.blockUntil > nowMs
17592
19259
  ? budget.blockUntil
17593
19260
  : observer?.nextAt ?? decision.retryAt ?? wakes.controlAt;
@@ -17607,13 +19274,14 @@ function App({ onCreateRemote = () => {} } = {}) {
17607
19274
  states: Object.fromEntries(TAB_KEYS.map((key) => [key, {
17608
19275
  unchangedCount: unchangedPolls[key],
17609
19276
  inProgressCI: key === "actions" && actionsInProgress(),
19277
+ requested: requestedTabs.has(key),
17610
19278
  }])),
17611
19279
  });
17612
19280
  scheduledWakePolls = planned.due;
17613
19281
  pollDueAt = planned.dueAt;
17614
19282
  backgroundIndex = planned.backgroundIndex;
17615
19283
  const outcomes = await Promise.allSettled(
17616
- planned.due.map(({ key, kind }) => requestTab(key, kind)),
19284
+ planned.due.map(({ key, kind }) => requestTab(key, kind, { detachTransport: true })),
17617
19285
  );
17618
19286
  const retryAt = governorControlRetryAt(Date.now(), runtime.refreshMs);
17619
19287
  const retried = retryFailedPollBatch({ due: planned.due, outcomes,
@@ -17666,6 +19334,13 @@ function App({ onCreateRemote = () => {} } = {}) {
17666
19334
 
17667
19335
  async function runControlWake() {
17668
19336
  if (cancelled) return;
19337
+ const controlAt = Date.now();
19338
+ const clockCause = wallClockRecovery(lastWallAt, controlAt, Math.min(runtime.refreshMs, 5_000));
19339
+ lastWallAt = controlAt;
19340
+ if (clockCause) {
19341
+ recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
19342
+ setNow(new Date(controlAt));
19343
+ }
17669
19344
  for (const item of [...cleanupQueue]) retryPendingCleanup(item.cleanupKey, item);
17670
19345
  await runtimeIdentityCoordinator.refresh();
17671
19346
  if (cancelled) return;
@@ -17684,11 +19359,11 @@ function App({ onCreateRemote = () => {} } = {}) {
17684
19359
  }
17685
19360
  const refreshed = currentScope
17686
19361
  ? await refreshRuntimeBudget(currentScope)
17687
- : { ok: false, reason: runtimeIdentityCoordinator?.inspect()?.reason ?? "stale" };
19362
+ : { ok: false, ...(runtimeIdentityCoordinator?.inspect() ?? { reason: "stale" }) };
17688
19363
  if (cancelled) return;
17689
19364
  const checkedAt = Date.now();
17690
19365
  const snapshot = currentScope ? inspectGovernor(currentScope, checkedAt) : refreshed;
17691
- if (currentScope) reconcileRuntimeUncertainty(currentScope, snapshot);
19366
+ if (currentScope) syncRuntimeQuotaProjection(currentScope, snapshot);
17692
19367
  if (currentScope && snapshot.ok && pendingBlockPublications.size > 0) {
17693
19368
  attemptPendingBlockPublications(currentScope, checkedAt, snapshot.value);
17694
19369
  }
@@ -17750,14 +19425,27 @@ function App({ onCreateRemote = () => {} } = {}) {
17750
19425
  const livenessWake = createRecurringWake({
17751
19426
  scheduler: wakeScheduler,
17752
19427
  intervalMs: Math.min(runtime.refreshMs, 5000),
17753
- run: async () => {
19428
+ run: () => {
17754
19429
  if (cancelled || remoteSetupRef.current) return;
17755
19430
  const at = Date.now();
19431
+ const clockCause = wallClockRecovery(lastWallAt, at, Math.min(runtime.refreshMs, 5_000));
19432
+ lastWallAt = at;
19433
+ for (const resource of RATE_RESOURCES) {
19434
+ if (["receipt-retry", "request-queue"].includes(recoveryRef.current[resource]?.code)) {
19435
+ clearRuntimeRecovery(resource);
19436
+ }
19437
+ }
19438
+ if (clockCause) {
19439
+ recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
19440
+ setNow(new Date(at));
19441
+ void controlWake();
19442
+ return;
19443
+ }
17756
19444
  if (wakeScheduler.at("control") <= at ||
17757
- !Number.isFinite(wakeScheduler.at("control"))) await controlWake();
19445
+ !Number.isFinite(wakeScheduler.at("control"))) void controlWake();
17758
19446
  if (pending.size > 0 || cleanupQueue.size > 0 ||
17759
19447
  wakeScheduler.at("data") <= Date.now() ||
17760
- Object.values(pollDueAt).some((due) => due <= Date.now())) await dataWake();
19448
+ Object.values(pollDueAt).some((due) => due <= Date.now())) void dataWake();
17761
19449
  },
17762
19450
  onError: (error) => {
17763
19451
  if (!cancelled) pauseCoordination(TABS[activeIndexRef.current].key, error?.reason ?? "stale");
@@ -17773,7 +19461,9 @@ function App({ onCreateRemote = () => {} } = {}) {
17773
19461
  if (cancelled) return;
17774
19462
  const currentScope = ensureScope(Date.now());
17775
19463
  if (!currentScope) {
17776
- pauseCoordination(TABS[activeIndexRef.current].key, runtimeIdentityCoordinator?.inspect()?.reason ?? "unknown-scope");
19464
+ const identityFailure = runtimeIdentityCoordinator?.inspect();
19465
+ pauseCoordination(TABS[activeIndexRef.current].key,
19466
+ identityFailure?.reason ?? "unknown-scope", null, false, identityFailure?.retryAt);
17777
19467
  armWake("control", governorControlRetryAt(Date.now(), runtime.refreshMs), controlWake);
17778
19468
  return;
17779
19469
  }
@@ -17783,7 +19473,7 @@ function App({ onCreateRemote = () => {} } = {}) {
17783
19473
  if (cancelled) return;
17784
19474
  const checkedAt = Date.now();
17785
19475
  const snapshot = inspectGovernor(currentScope, checkedAt);
17786
- reconcileRuntimeUncertainty(currentScope, snapshot);
19476
+ syncRuntimeQuotaProjection(currentScope, snapshot);
17787
19477
  publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
17788
19478
  if (snapshot.ok) {
17789
19479
  controlEpochs = { ...snapshot.value.epochs };
@@ -17907,7 +19597,14 @@ function App({ onCreateRemote = () => {} } = {}) {
17907
19597
  const displayError = formatTabErrorForWidth(tabError, failureContext, Math.max(1, cols - 5));
17908
19598
  const coordinationError = activeGovernorDecision?.coordinationError && !remoteSetup;
17909
19599
  const coordinationReason = activeFailure ?? activeGovernorDecision?.reason ?? "unavailable";
17910
- const coordinationCondition = coordinationError ? `${tab.key}\0${coordinationReason}` : null;
19600
+ const noticeReason = activeRecovery?.code ??
19601
+ (tabError?.kind === "fetch" ? tabError.verdict : null) ??
19602
+ (coordinationError || activeGovernorDecision?.mode === "paused" ? coordinationReason : null) ??
19603
+ (statusDisconnected ? runtime.connect === null ? "identity-unavailable"
19604
+ : activeGovernorDecision?.reason === "app-auth" ? "app-auth" : "disconnected" : null);
19605
+ // A continuous fault stays sustained when its explanation becomes more
19606
+ // specific. Only losing the fault (or switching tabs) restarts the debounce.
19607
+ const coordinationCondition = !remoteSetup && noticeReason ? tab.key : null;
17911
19608
  useEffect(() => {
17912
19609
  setVisibleCoordinationCondition(null);
17913
19610
  if (coordinationCondition === null) return;
@@ -17919,10 +19616,21 @@ function App({ onCreateRemote = () => {} } = {}) {
17919
19616
  }, [coordinationCondition]);
17920
19617
  const showCoordinationNotice = coordinationCondition !== null &&
17921
19618
  visibleCoordinationCondition === coordinationCondition;
17922
- const noticeLine = showCoordinationNotice
17923
- ? activeFailure ? acquisitionFailureCopy(activeFailure)[cols < 40 ? "narrow" : "full"]
17924
- : coordinationNotice(coordinationReason)
17925
- : !remoteSetup && displayError ? displayError : "";
19619
+ const noticeResource = activeResource;
19620
+ const noticeSourceAt = collectorSourceAgeRef.current[tab.key]
19621
+ ? now.getTime() - collectorSourceAgeRef.current[tab.key].ageMs -
19622
+ Math.max(0, performance.now() - collectorSourceAgeRef.current[tab.key].measuredAt)
19623
+ : lastOkRef.current[tab.key] ?? null;
19624
+ const noticeCause = activeRecovery ?? (noticeReason ? recoveryCause({
19625
+ reason: noticeReason === "other" ? "network-outage" : noticeReason,
19626
+ resource: noticeResource, sourceAt: noticeSourceAt,
19627
+ retryAt: activeGovernorDecision?.retryAt ?? activeGovernorDecision?.resetMs ??
19628
+ activeGovernorDecision?.notBefore ?? null,
19629
+ origin: tabError ? "source" : "coordination", tab: tabError ? tab.key : null,
19630
+ }) : null);
19631
+ const noticeLines = showCoordinationNotice && noticeCause
19632
+ ? presentRecovery(noticeCause, { nowMs: now.getTime(), cols })
19633
+ : [!remoteSetup && displayError ? displayError : ""];
17926
19634
  const noticeTone = showCoordinationNotice ? ATTENTION : displayError ? ERROR_TEXT : undefined;
17927
19635
  const spin = SPINNER[frame % SPINNER.length];
17928
19636
 
@@ -17999,6 +19707,22 @@ function App({ onCreateRemote = () => {} } = {}) {
17999
19707
  });
18000
19708
  }
18001
19709
 
19710
+ if (runtime.background === "on-demand" && tab.key !== "actions" && data[tab.key] !== null &&
19711
+ ["watching", "shared", "stale"].includes(semanticStatus.kind) && !tabError &&
19712
+ !activeFailure && !(tab.key === "security" && securityBlind)) {
19713
+ semanticStatus = refreshStatus({ onDemand: true });
19714
+ }
19715
+
19716
+ // A settled on-demand view has no data wake to advance its visible age.
19717
+ // Tick only its display clock; source timestamps and acquisition stay idle.
19718
+ const showCachedAge = semanticStatus.kind === "cached" && staleFor !== null;
19719
+ useEffect(() => {
19720
+ if (!showCachedAge) return;
19721
+ setNow(new Date());
19722
+ const timer = setInterval(() => setNow(new Date()), 1000);
19723
+ return () => clearInterval(timer);
19724
+ }, [showCachedAge]);
19725
+
18002
19726
  const allItems = items ?? [];
18003
19727
  const tabOffsetRaw = offset[tab.key] ?? 0;
18004
19728
  const selectedKey = selected[tab.key] ?? null;
@@ -18199,7 +19923,8 @@ function App({ onCreateRemote = () => {} } = {}) {
18199
19923
  !showHelp &&
18200
19924
  tooNarrow &&
18201
19925
  e(Text, { dimColor: true, wrap: "truncate-end" }, "too narrow"),
18202
- e(Text, { color: noticeTone, wrap: "truncate-end" }, noticeLine),
19926
+ ...noticeLines.map((line, index) =>
19927
+ e(Text, { key: `recovery:${index}`, color: noticeTone, wrap: "truncate-end" }, line)),
18203
19928
  !showHelp &&
18204
19929
  !tooNarrow &&
18205
19930
  !remoteSetup &&
@@ -18294,7 +20019,8 @@ function App({ onCreateRemote = () => {} } = {}) {
18294
20019
  status: semanticStatus,
18295
20020
  detail: activeGovernorDecision,
18296
20021
  spin: showSpinner ? spin : null,
18297
- stale: staleLabel,
20022
+ stale: semanticStatus.kind === "cached" && staleFor != null
20023
+ ? formatDuration(Math.max(0, staleFor)) : staleLabel,
18298
20024
  nowMs: now.getTime(),
18299
20025
  interactive,
18300
20026
  cols: frameCols,
@@ -18589,6 +20315,7 @@ export {
18589
20315
  resolveEffectiveCredential,
18590
20316
  identityRegistryRoot,
18591
20317
  inspectIdentityRegistry,
20318
+ doctorCachedQuotaScope,
18592
20319
  claimIdentityBootstrap,
18593
20320
  finishIdentityBootstrap,
18594
20321
  createIdentityCoordinator,
@@ -18628,11 +20355,17 @@ export {
18628
20355
  GOVERNOR_PHASE_WINDOW_MS,
18629
20356
  BUDGET_PROBE_MS,
18630
20357
  MIN_SAMPLE_CALLS,
20358
+ MAX_PACING_EXTERNAL_FACTOR,
20359
+ IDENTITY_UNCERTAIN_MAX_MS,
18631
20360
  REQUEST_PRIORITIES,
18632
20361
  normalizeBudgetResource,
18633
20362
  budgetEpoch,
18634
20363
  resourceReserve,
18635
20364
  availableForGrant,
20365
+ governorChargedCost,
20366
+ governorAvailableForGrant,
20367
+ governorResourceDecision,
20368
+ governorResourceHeld,
18636
20369
  nextExternalFactor,
18637
20370
  resourceDecision,
18638
20371
  governorPhaseOffset,
@@ -18658,6 +20391,7 @@ export {
18658
20391
  serializeGovernorState,
18659
20392
  readGovernorState,
18660
20393
  writeGovernorState,
20394
+ writeGovernorQuotaState,
18661
20395
  pidIsDead,
18662
20396
  claimGovernorLock,
18663
20397
  releaseGovernorLock,
@@ -18685,6 +20419,9 @@ export {
18685
20419
  readIntentDecision,
18686
20420
  cancelIntent,
18687
20421
  startReservation,
20422
+ issueGovernorDispatch,
20423
+ terminalGovernorDispatch,
20424
+ acknowledgeSealedGeneration,
18688
20425
  completeReservation,
18689
20426
  settleReservationWithBudgetObservations,
18690
20427
  recordResourceBlock,
@@ -18846,7 +20583,9 @@ export {
18846
20583
  acquisitionRequestMetrics,
18847
20584
  acquisitionFailureHold,
18848
20585
  setRuntimeAcquisitionHold,
20586
+ clearScheduledRuntimeControlHold,
18849
20587
  loadAcquisitionStore,
20588
+ readAcquisitionSnapshot,
18850
20589
  createAcquisitionEngine,
18851
20590
  pollPolicyInterval,
18852
20591
  advanceUnchangedCount,
@@ -18882,11 +20621,27 @@ export {
18882
20621
  runtimeIntentGate,
18883
20622
  rateLimitBlockDecision,
18884
20623
  coordinationNotice,
20624
+ recoveryCause,
20625
+ shouldClearRecoveryCause,
20626
+ recoveryRetryAt,
20627
+ terminalizationRecoveryCode,
20628
+ terminalizationBacklogForResource,
20629
+ receiptCapabilityFromReservation,
20630
+ requestFailureRecoveryCode,
20631
+ requestFailureRecovery,
20632
+ wallClockRecovery,
20633
+ collectorSourceAdvanced,
20634
+ cleanupQueueForResource,
20635
+ presentRecovery,
20636
+ recordRecoveryTransition,
20637
+ safeRecordRecoveryTransition,
20638
+ readRecoveryTransitions,
18885
20639
  mergeRateLimitBlockPublications,
18886
20640
  hydrateRateLimitBlockPublication,
18887
20641
  retryRateLimitBlockPublication,
18888
20642
  rateLimitBlockProbeRecovered,
18889
20643
  admitGovernorOperation,
20644
+ queueTerminalization,
18890
20645
  runAdmittedOperation,
18891
20646
  abortableDelay,
18892
20647
  awaitCollectorReservation,
@@ -18912,6 +20667,7 @@ export {
18912
20667
  demandedPageCount,
18913
20668
  mergeDemandedPages,
18914
20669
  operationPausedUntil,
20670
+ pauseOperation,
18915
20671
  LIST_LIMIT,
18916
20672
  readGraphqlObserver,
18917
20673
  GRAPHQL_QUERIES,