gh-glance 0.15.2 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +52 -1
- package/README.md +76 -25
- package/index.mjs +2237 -500
- package/package.json +1 -1
package/index.mjs
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
// disarmDevBuildLeak() for how that path is kept survivable rather than fatal.
|
|
23
23
|
process.env.NODE_ENV ??= "production";
|
|
24
24
|
|
|
25
|
-
import { execFile, spawn } from "node:child_process";
|
|
25
|
+
import { execFile, execFileSync, spawn } from "node:child_process";
|
|
26
26
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
27
27
|
import { createHash, createHmac, createPrivateKey, randomUUID, sign as cryptoSign, timingSafeEqual } from "node:crypto";
|
|
28
28
|
import { EventEmitter } from "node:events";
|
|
@@ -207,7 +207,7 @@ const STALE_AFTER_MS = 30_000;
|
|
|
207
207
|
// A coordination hold must persist long enough to be meaningful before it
|
|
208
208
|
// takes the notice row. Short startup and lock-contention states recover on
|
|
209
209
|
// their own and should not present themselves as user-visible failures.
|
|
210
|
-
const COORDINATION_NOTICE_AFTER_MS =
|
|
210
|
+
const COORDINATION_NOTICE_AFTER_MS = 800;
|
|
211
211
|
|
|
212
212
|
// Mutable because the fetchers below are defined before argv is parsed, and the
|
|
213
213
|
// argv block is what fills this in. Everything here has a working default, so
|
|
@@ -221,7 +221,7 @@ const runtime = {
|
|
|
221
221
|
host: null,
|
|
222
222
|
repoExplicit: false,
|
|
223
223
|
refreshMs: REFRESH_MS,
|
|
224
|
-
background: "
|
|
224
|
+
background: "on-demand",
|
|
225
225
|
verbose: false,
|
|
226
226
|
connect: null,
|
|
227
227
|
initialTabIndex: 0,
|
|
@@ -734,7 +734,9 @@ const pausedOperations = new Map();
|
|
|
734
734
|
|
|
735
735
|
function pauseOperation(operation, untilMs) {
|
|
736
736
|
if (!Number.isFinite(untilMs)) return;
|
|
737
|
-
|
|
737
|
+
const until = Math.max(pausedOperations.get(operation) ?? 0, untilMs);
|
|
738
|
+
pausedOperations.set(operation, until);
|
|
739
|
+
return () => { if (pausedOperations.get(operation) === until) pausedOperations.delete(operation); };
|
|
738
740
|
}
|
|
739
741
|
|
|
740
742
|
function operationPausedUntil(operation, now = Date.now()) {
|
|
@@ -744,7 +746,8 @@ function operationPausedUntil(operation, now = Date.now()) {
|
|
|
744
746
|
return until;
|
|
745
747
|
}
|
|
746
748
|
|
|
747
|
-
async function runGh(args, { signal, operation, input = null, execute = null
|
|
749
|
+
async function runGh(args, { signal, operation, input = null, execute = null,
|
|
750
|
+
closeWaitMs = 15_000 } = {}) {
|
|
748
751
|
if (operationCost(operation) === null) {
|
|
749
752
|
throw new Error(`undeclared gh operation: ${operation ?? "missing"}`);
|
|
750
753
|
}
|
|
@@ -756,11 +759,32 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
756
759
|
const requestNow = typeof bound?.now === "function" ? bound.now : Date.now;
|
|
757
760
|
const permit = identityCoordinator && !local
|
|
758
761
|
? await acquireIdentityHttpPermit(identityCoordinator, { signal, now: requestNow,
|
|
762
|
+
operationDeadline: CONTROL_OPERATIONS.includes(operation) ? undefined : bound?.receiptCapability?.deadline,
|
|
759
763
|
...(typeof bound?.httpWait === "function" ? { wait: bound.httpWait } : {}) }) : null;
|
|
760
764
|
const startedAt = requestNow();
|
|
761
765
|
let requestError = null;
|
|
762
766
|
let requestStdout = null;
|
|
763
767
|
let requestStarted = false;
|
|
768
|
+
let knownNeverStarted = false;
|
|
769
|
+
let requestSettled = false;
|
|
770
|
+
let successfulPromise = false;
|
|
771
|
+
let dispatchEvidence = null;
|
|
772
|
+
let dispatchCapability = null;
|
|
773
|
+
let childCloseProof = null;
|
|
774
|
+
let activeDispatchKey = null;
|
|
775
|
+
let activeDispatchReleased = false;
|
|
776
|
+
const releaseActiveDispatch = () => {
|
|
777
|
+
if (!activeDispatchKey || activeDispatchReleased) return;
|
|
778
|
+
activeDispatchReleased = true;
|
|
779
|
+
const remaining = (activeOwnerDispatches.get(activeDispatchKey) ?? 1) - 1;
|
|
780
|
+
if (remaining > 0) activeOwnerDispatches.set(activeDispatchKey, remaining);
|
|
781
|
+
else activeOwnerDispatches.delete(activeDispatchKey);
|
|
782
|
+
if (dispatchCapability && !dispatchCapability.controlResource) {
|
|
783
|
+
const sealed = inspectGovernor({ ...bound, identityProvider: null }, requestNow());
|
|
784
|
+
const nonce = sealed.ok && sealed.value.debtGroups[dispatchCapability.generation]?.nonce;
|
|
785
|
+
if (nonce) acknowledgeSealedGeneration(bound, dispatchCapability, nonce, requestNow());
|
|
786
|
+
}
|
|
787
|
+
};
|
|
764
788
|
try {
|
|
765
789
|
assertBoundCredential(bound);
|
|
766
790
|
if (identityCoordinator && operation === "budget-core-observer") {
|
|
@@ -789,6 +813,23 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
789
813
|
}
|
|
790
814
|
}
|
|
791
815
|
assertBoundCredential(bound);
|
|
816
|
+
if (!local && RATE_RESOURCES.some((resource) => operationCost(operation)[resource] > 0)) {
|
|
817
|
+
dispatchCapability = operation === "budget-core-observer"
|
|
818
|
+
? control?.value?.receiptCapability
|
|
819
|
+
: operation === "graphql-observer"
|
|
820
|
+
? bound?.controlReceiptCapability : bound?.receiptCapability;
|
|
821
|
+
if (dispatchCapability) {
|
|
822
|
+
activeDispatchKey = ownerDispatchKey(bound, dispatchCapability);
|
|
823
|
+
activeOwnerDispatches.set(activeDispatchKey,
|
|
824
|
+
(activeOwnerDispatches.get(activeDispatchKey) ?? 0) + 1);
|
|
825
|
+
}
|
|
826
|
+
const issued = issueGovernorDispatch(bound, dispatchCapability, operation, requestNow());
|
|
827
|
+
if (!issued.ok) {
|
|
828
|
+
releaseActiveDispatch();
|
|
829
|
+
throw new Error(`API receipt paused (${issued.reason})`);
|
|
830
|
+
}
|
|
831
|
+
dispatchEvidence = issued.value;
|
|
832
|
+
}
|
|
792
833
|
const executeChild = execute ?? bound?.executeGh ?? execFileAsync;
|
|
793
834
|
const pending = executeChild("gh", args, {
|
|
794
835
|
timeout: GH_TIMEOUT_MS,
|
|
@@ -798,6 +839,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
798
839
|
signal,
|
|
799
840
|
});
|
|
800
841
|
requestStarted = true;
|
|
842
|
+
if (Number.isSafeInteger(pending.child?.pid) &&
|
|
843
|
+
typeof pending.child.once === "function") {
|
|
844
|
+
childCloseProof = new Promise((resolve) => pending.child.once("close", resolve));
|
|
845
|
+
}
|
|
846
|
+
if (dispatchEvidence && Number.isSafeInteger(pending.child?.pid)) {
|
|
847
|
+
markGovernorDispatchChild(bound, dispatchCapability,
|
|
848
|
+
dispatchEvidence.sequence, pending.child.pid, requestNow());
|
|
849
|
+
}
|
|
801
850
|
if (typeof input === "string") {
|
|
802
851
|
// A child killed by the timeout or the abort signal closes stdin under
|
|
803
852
|
// us; that EPIPE is the kill's consequence, not the failure worth
|
|
@@ -805,16 +854,73 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
805
854
|
pending.child.stdin?.on("error", () => {});
|
|
806
855
|
pending.child.stdin?.end(input);
|
|
807
856
|
}
|
|
808
|
-
|
|
857
|
+
let stdout;
|
|
858
|
+
try {
|
|
859
|
+
({ stdout } = await pending);
|
|
860
|
+
requestSettled = true;
|
|
861
|
+
successfulPromise = true;
|
|
862
|
+
} catch (error) {
|
|
863
|
+
requestSettled = true;
|
|
864
|
+
throw error;
|
|
865
|
+
}
|
|
809
866
|
requestStdout = stdout;
|
|
810
|
-
logGh(args, startedAt, `ok ${stdout.length}B`);
|
|
811
|
-
return stdout;
|
|
812
867
|
} catch (err) {
|
|
813
868
|
requestError = err;
|
|
869
|
+
if (!requestStarted && !dispatchEvidence) err.notStarted ??= true;
|
|
870
|
+
knownNeverStarted = !requestStarted && err?.notStarted === true;
|
|
814
871
|
if (requestStarted && isRecord(err)) err.httpStarted = true;
|
|
815
872
|
logGh(args, startedAt, `FAILED ${shortErr(err)}`);
|
|
816
873
|
throw err;
|
|
817
874
|
} finally {
|
|
875
|
+
let childClosed = !requestStarted;
|
|
876
|
+
if (requestSettled && childCloseProof) {
|
|
877
|
+
let timer;
|
|
878
|
+
childClosed = await Promise.race([
|
|
879
|
+
childCloseProof.then(() => true),
|
|
880
|
+
new Promise((resolve) => { timer = setTimeout(() => resolve(false), closeWaitMs); }),
|
|
881
|
+
]);
|
|
882
|
+
clearTimeout(timer);
|
|
883
|
+
}
|
|
884
|
+
if (requestSettled && !childClosed) {
|
|
885
|
+
requestError ??= Object.assign(new Error("GitHub child close unverified"), { httpStarted: true });
|
|
886
|
+
}
|
|
887
|
+
if (successfulPromise) {
|
|
888
|
+
logGh(args, startedAt, childClosed
|
|
889
|
+
? `ok ${requestStdout.length}B` : "FAILED GitHub child close unverified");
|
|
890
|
+
}
|
|
891
|
+
if (dispatchEvidence && (requestSettled && childClosed || knownNeverStarted)) {
|
|
892
|
+
const finishDispatch = () => {
|
|
893
|
+
const result = terminalGovernorDispatch(bound,
|
|
894
|
+
dispatchCapability, dispatchEvidence.sequence, requestNow(),
|
|
895
|
+
{ neverStarted: knownNeverStarted });
|
|
896
|
+
if (result.ok || result.reason === "compacted") releaseActiveDispatch();
|
|
897
|
+
return result;
|
|
898
|
+
};
|
|
899
|
+
const terminal = await retryIdentityCompletion(finishDispatch, {
|
|
900
|
+
timeoutMs: 15_000, shouldRetry: () => true,
|
|
901
|
+
});
|
|
902
|
+
if (!terminal.ok && identityCoordinator?.deferCompletion) {
|
|
903
|
+
identityCoordinator.deferCompletion(
|
|
904
|
+
`dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
|
|
905
|
+
finishDispatch,
|
|
906
|
+
);
|
|
907
|
+
}
|
|
908
|
+
} else if (dispatchEvidence && requestSettled && childCloseProof) {
|
|
909
|
+
// A promise can settle before its subprocess closes. Keep the issued
|
|
910
|
+
// charge until close proves that no more HTTP can escape this child.
|
|
911
|
+
void childCloseProof.then(async () => {
|
|
912
|
+
const finishDispatch = () => terminalGovernorDispatch(bound,
|
|
913
|
+
dispatchCapability, dispatchEvidence.sequence, requestNow());
|
|
914
|
+
const terminal = await retryIdentityCompletion(finishDispatch, {
|
|
915
|
+
timeoutMs: 15_000, shouldRetry: () => true,
|
|
916
|
+
});
|
|
917
|
+
if (terminal.ok || terminal.reason === "compacted") releaseActiveDispatch();
|
|
918
|
+
else identityCoordinator?.deferCompletion?.(
|
|
919
|
+
`dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
|
|
920
|
+
finishDispatch,
|
|
921
|
+
);
|
|
922
|
+
});
|
|
923
|
+
}
|
|
818
924
|
if (permit) {
|
|
819
925
|
const release = () => releaseIdentityHttpPermit(identityCoordinator, permit, requestError, requestNow());
|
|
820
926
|
const released = await retryIdentityCompletion(release, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
|
|
@@ -822,11 +928,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
822
928
|
}
|
|
823
929
|
if (control?.ok) {
|
|
824
930
|
const settle = () => settleIdentityControl(identityCoordinator, control.value,
|
|
825
|
-
requestStdout ?? requestError?.stdout, requestNow()
|
|
931
|
+
childClosed ? requestStdout ?? requestError?.stdout : null, requestNow(),
|
|
932
|
+
{ neverStarted: !requestStarted && (!dispatchEvidence || knownNeverStarted) });
|
|
826
933
|
const settled = await retryIdentityCompletion(settle, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
|
|
827
934
|
if (!settled.ok) identityCoordinator.deferCompletion(`control:${control.value.id}`, settle);
|
|
828
935
|
}
|
|
829
936
|
}
|
|
937
|
+
if (successfulPromise && requestError) throw requestError;
|
|
938
|
+
return requestStdout;
|
|
830
939
|
}
|
|
831
940
|
|
|
832
941
|
function parseGhApiResponse(stdout) {
|
|
@@ -1638,7 +1747,6 @@ async function fetchActions(signal, {
|
|
|
1638
1747
|
}
|
|
1639
1748
|
const responses = [runsResponse];
|
|
1640
1749
|
let requestMetrics = runsResponse.requestMetrics ?? restResponseRequestMetrics(runsResponse);
|
|
1641
|
-
const uncertainReceipts = [];
|
|
1642
1750
|
const runs = parseActionsRuns(runsResponse.body);
|
|
1643
1751
|
const startedAt = Date.now();
|
|
1644
1752
|
let nextCatalog = catalog;
|
|
@@ -1662,7 +1770,6 @@ async function fetchActions(signal, {
|
|
|
1662
1770
|
request,
|
|
1663
1771
|
}),
|
|
1664
1772
|
});
|
|
1665
|
-
uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
|
|
1666
1773
|
if (admitted.ok && admitted.value) {
|
|
1667
1774
|
responses.push(admitted.value);
|
|
1668
1775
|
requestMetrics = mergeAcquisitionRequestMetrics(
|
|
@@ -1695,7 +1802,6 @@ async function fetchActions(signal, {
|
|
|
1695
1802
|
failedRequests: requestMetrics.failedRequests ?? 0,
|
|
1696
1803
|
coreUnitsTotal: requestMetrics.coreUnits ?? 0,
|
|
1697
1804
|
requestMetrics,
|
|
1698
|
-
uncertainReceipts,
|
|
1699
1805
|
graphqlSpent: GRAPHQL_PER_FETCH.actions,
|
|
1700
1806
|
stagedEntities: batch.stagedEntities,
|
|
1701
1807
|
observations: batch.observations,
|
|
@@ -1896,7 +2002,6 @@ async function fetchGraphqlList(kind, mapRow, {
|
|
|
1896
2002
|
throw error;
|
|
1897
2003
|
}
|
|
1898
2004
|
const observations = [...first.observations];
|
|
1899
|
-
const uncertainReceipts = [];
|
|
1900
2005
|
const repositoryIdentity = normalizeRepositoryIdentity(first.data?.repository);
|
|
1901
2006
|
// Only the first page. Pages past it opened their own reservation inside
|
|
1902
2007
|
// runAdmittedOperation and settle against it, so adding them here charges the
|
|
@@ -1934,7 +2039,6 @@ async function fetchGraphqlList(kind, mapRow, {
|
|
|
1934
2039
|
waitMs: GOVERNOR_ADMISSION_WAIT_MS,
|
|
1935
2040
|
run: (admittedSignal) => fetchPage(kind, { signal: admittedSignal, after: cursor, operation: `page:${kind}` }),
|
|
1936
2041
|
});
|
|
1937
|
-
uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
|
|
1938
2042
|
// A later page that is denied, fails, or returns errors leaves the rows
|
|
1939
2043
|
// already gathered exactly as they are. Losing page one because page two
|
|
1940
2044
|
// was refused would turn a budget decision into data loss.
|
|
@@ -1988,7 +2092,6 @@ async function fetchGraphqlList(kind, mapRow, {
|
|
|
1988
2092
|
httpRequests: requestMetrics.httpRequests ?? 0,
|
|
1989
2093
|
failedRequests: requestMetrics.failedRequests ?? 0,
|
|
1990
2094
|
requestMetrics,
|
|
1991
|
-
uncertainReceipts,
|
|
1992
2095
|
observations,
|
|
1993
2096
|
incomplete,
|
|
1994
2097
|
repositoryIdentity,
|
|
@@ -2265,6 +2368,41 @@ function reservationCost(reservation, resource, leases, nowMs) {
|
|
|
2265
2368
|
return Number.isFinite(cost) && cost > 0 ? Math.max(0, cost - accounted) : 0;
|
|
2266
2369
|
}
|
|
2267
2370
|
|
|
2371
|
+
function governorAccountingSafe(state, nowMs) {
|
|
2372
|
+
return RATE_RESOURCES.every((resource) =>
|
|
2373
|
+
Number.isSafeInteger(governorChargedCost(state, resource, nowMs)));
|
|
2374
|
+
}
|
|
2375
|
+
|
|
2376
|
+
function governorChargedCost(state, resource, nowMs) {
|
|
2377
|
+
if (!state || !RATE_RESOURCES.includes(resource)) return Number.NaN;
|
|
2378
|
+
let total = state.debt?.[resource]?.unresolvedUnits +
|
|
2379
|
+
state.debt?.[resource]?.quiescentUnits;
|
|
2380
|
+
if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
|
|
2381
|
+
for (const reservation of [state.controlReceipts?.[resource],
|
|
2382
|
+
...Object.values(state.reservations ?? {})]) {
|
|
2383
|
+
total += reservationCost(reservation, resource, state.leases, nowMs);
|
|
2384
|
+
if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
|
|
2385
|
+
}
|
|
2386
|
+
return total;
|
|
2387
|
+
}
|
|
2388
|
+
|
|
2389
|
+
function governorAvailableForGrant(state, resource, nowMs) {
|
|
2390
|
+
return availableForGrant({ budget: state?.budgets?.[resource], resource, nowMs,
|
|
2391
|
+
chargedCost: governorChargedCost(state, resource, nowMs) });
|
|
2392
|
+
}
|
|
2393
|
+
|
|
2394
|
+
function governorResourceDecision(state, resource, nowMs, cost) {
|
|
2395
|
+
return resourceDecision({ budget: state?.budgets?.[resource], resource, nowMs, cost,
|
|
2396
|
+
chargedCost: governorChargedCost(state, resource, nowMs) });
|
|
2397
|
+
}
|
|
2398
|
+
|
|
2399
|
+
function governorResourceHeld(state, resource, nowMs, pendingBlock = false) {
|
|
2400
|
+
const budget = state?.budgets?.[resource];
|
|
2401
|
+
const decision = governorAvailableForGrant(state, resource, nowMs);
|
|
2402
|
+
return pendingBlock || budget?.blockUntil > nowMs ||
|
|
2403
|
+
decision.mode === "paused" || decision.mode === "open" && decision.spendable <= 0;
|
|
2404
|
+
}
|
|
2405
|
+
|
|
2268
2406
|
function availableForGrant({
|
|
2269
2407
|
budget,
|
|
2270
2408
|
resource = budget?.resource,
|
|
@@ -2517,6 +2655,7 @@ function scheduleIntents({
|
|
|
2517
2655
|
leases = {},
|
|
2518
2656
|
budgets = {},
|
|
2519
2657
|
reservations = [],
|
|
2658
|
+
debtCosts = { core: 0, graphql: 0 },
|
|
2520
2659
|
lanes = {},
|
|
2521
2660
|
cursors = {},
|
|
2522
2661
|
nowMs,
|
|
@@ -2551,7 +2690,7 @@ function scheduleIntents({
|
|
|
2551
2690
|
|
|
2552
2691
|
const chargedTotals = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
2553
2692
|
resource,
|
|
2554
|
-
reservations.reduce(
|
|
2693
|
+
debtCosts[resource] + reservations.reduce(
|
|
2555
2694
|
(total, reservation) => total + reservationCost(reservation, resource, leases, nowMs),
|
|
2556
2695
|
0,
|
|
2557
2696
|
),
|
|
@@ -2694,17 +2833,17 @@ function scheduleIntents({
|
|
|
2694
2833
|
}
|
|
2695
2834
|
|
|
2696
2835
|
// ---------- Credential identities and restart-safe coordination ----------
|
|
2697
|
-
const IDENTITY_VERSION =
|
|
2836
|
+
const IDENTITY_VERSION = 2;
|
|
2698
2837
|
const IDENTITY_ATTEMPT_WINDOW_MS = 15 * 60_000;
|
|
2699
2838
|
const IDENTITY_ATTEMPT_LIMIT = 3;
|
|
2700
2839
|
const IDENTITY_HOST_ATTEMPT_LIMIT = 12;
|
|
2701
2840
|
const IDENTITY_MAX_ATTEMPTS = 4096;
|
|
2702
2841
|
const HTTP_START_GAP_MS = 250;
|
|
2703
2842
|
const HTTP_MAX_WAITERS = 128;
|
|
2704
|
-
|
|
2705
|
-
//
|
|
2706
|
-
//
|
|
2707
|
-
//
|
|
2843
|
+
const HTTP_MAX_PERMITS = 3;
|
|
2844
|
+
// Retain the existing bounded permit lease for managed requests. This bounds
|
|
2845
|
+
// unexpired admissions, not surviving OS children after suspension or owner
|
|
2846
|
+
// death; quota receipts separately retain uncertain transport charges.
|
|
2708
2847
|
const HTTP_PERMIT_MAX_MS = GH_TIMEOUT_MS + 5_000;
|
|
2709
2848
|
// The longest a primary window can run. An attempt that never obtained reset
|
|
2710
2849
|
// evidence is still covered once a full window has demonstrably elapsed since it
|
|
@@ -2738,11 +2877,15 @@ function retryableCoordination(reason) {
|
|
|
2738
2877
|
return RETRYABLE_COORDINATION_REASONS.includes(reason);
|
|
2739
2878
|
}
|
|
2740
2879
|
|
|
2741
|
-
//
|
|
2742
|
-
//
|
|
2743
|
-
// test a stopped process, or one whose release never won the lock, holds the
|
|
2744
|
-
// single machine-wide permit forever and every pane queues behind it.
|
|
2880
|
+
// Managed permit leases retain their existing age/dead-owner reclamation law.
|
|
2881
|
+
// Reclaiming a slot never retires its quota receipt or proves child quiescence.
|
|
2745
2882
|
function staleHttpPermit(permit, now, kill = process.kill.bind(process)) {
|
|
2883
|
+
// A dead v1 parent does not prove its untracked child ended. Reserve its
|
|
2884
|
+
// one slot until a subsequent verified boot; two managed slots remain usable.
|
|
2885
|
+
if (permit.kind === "legacy") {
|
|
2886
|
+
const bootId = coordinationBootId();
|
|
2887
|
+
return permit.bootId !== null && bootId !== null && permit.bootId !== bootId;
|
|
2888
|
+
}
|
|
2746
2889
|
if (pidIsDead(permit.pid, kill)) return true;
|
|
2747
2890
|
return Number.isFinite(permit.startedAt) && now - permit.startedAt > HTTP_PERMIT_MAX_MS;
|
|
2748
2891
|
}
|
|
@@ -2813,8 +2956,8 @@ function emptyIdentityRegistry() {
|
|
|
2813
2956
|
return { version: IDENTITY_VERSION, migration: { activated: false, holdUntil: 0 }, identities: {}, attempts: {}, hosts: {} };
|
|
2814
2957
|
}
|
|
2815
2958
|
|
|
2816
|
-
function normalizeIdentityRegistry(raw) {
|
|
2817
|
-
if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !==
|
|
2959
|
+
function normalizeIdentityRegistry(raw, version = IDENTITY_VERSION) {
|
|
2960
|
+
if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !== version ||
|
|
2818
2961
|
!exactKeys(raw.migration, ["activated", "holdUntil"]) || typeof raw.migration.activated !== "boolean" ||
|
|
2819
2962
|
!Number.isFinite(raw.migration.holdUntil) || raw.migration.holdUntil < 0 ||
|
|
2820
2963
|
!isRecord(raw.identities) || !isRecord(raw.attempts) || !isRecord(raw.hosts) ||
|
|
@@ -2837,13 +2980,20 @@ function normalizeIdentityRegistry(raw) {
|
|
|
2837
2980
|
}
|
|
2838
2981
|
for (const [host, transport] of Object.entries(raw.hosts)) {
|
|
2839
2982
|
if (!isRecord(transport)) return null;
|
|
2840
|
-
const keys = ["lastStartedAt", "cooldownUntil", "permit"];
|
|
2983
|
+
const keys = ["lastStartedAt", "cooldownUntil", version === 1 ? "permit" : "permits"];
|
|
2841
2984
|
if (Object.hasOwn(transport, "waiters")) keys.push("waiters");
|
|
2842
2985
|
if (Object.hasOwn(transport, "throttle")) keys.push("throttle");
|
|
2843
2986
|
if (!normalizeHost(host) || !exactKeys(transport, keys) ||
|
|
2844
2987
|
!time(transport.lastStartedAt) || !time(transport.cooldownUntil)) return null;
|
|
2845
|
-
|
|
2846
|
-
|
|
2988
|
+
const permits = version === 1 ? (transport.permit === null ? [] : [transport.permit]) : transport.permits;
|
|
2989
|
+
if (!Array.isArray(permits) || permits.length > HTTP_MAX_PERMITS ||
|
|
2990
|
+
new Set(permits.map((permit) => permit?.nonce)).size !== permits.length ||
|
|
2991
|
+
permits.filter((permit) => permit?.kind === "legacy").length > 1 ||
|
|
2992
|
+
permits.some((permit) => !exactKeys(permit, version === 1
|
|
2993
|
+
? ["pid", "nonce", "startedAt"] : ["pid", "nonce", "startedAt", "kind", "bootId"]) ||
|
|
2994
|
+
!Number.isSafeInteger(permit.pid) || permit.pid < 1 || !validGovernorId(permit.nonce) || !time(permit.startedAt) ||
|
|
2995
|
+
version !== 1 && (!["request", "bootstrap", "legacy"].includes(permit.kind) ||
|
|
2996
|
+
permit.bootId !== null && (typeof permit.bootId !== "string" || permit.bootId.length > 128)))) return null;
|
|
2847
2997
|
if (!Object.hasOwn(transport, "throttle")) transport.throttle = emptyTransportThrottle();
|
|
2848
2998
|
if (!isRecord(transport.throttle) || !exactKeys(transport.throttle, ["attempts", "lastAt", "paused"]) ||
|
|
2849
2999
|
!Number.isSafeInteger(transport.throttle.attempts) || transport.throttle.attempts < 0 ||
|
|
@@ -2874,8 +3024,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
|
|
|
2874
3024
|
let state;
|
|
2875
3025
|
if (raw === null) state = emptyIdentityRegistry();
|
|
2876
3026
|
else {
|
|
2877
|
-
try {
|
|
2878
|
-
|
|
3027
|
+
try {
|
|
3028
|
+
const parsed = JSON.parse(raw);
|
|
3029
|
+
state = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
|
|
3030
|
+
} catch { return { ok: false, reason: "corrupt" }; }
|
|
2879
3031
|
}
|
|
2880
3032
|
if (!state) return { ok: false, reason: "corrupt" };
|
|
2881
3033
|
// Captured before the prunes below so their deletions still count as a
|
|
@@ -2883,6 +3035,19 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
|
|
|
2883
3035
|
// funnels through here, and each one used to serialize and rename the
|
|
2884
3036
|
// registry while holding the one machine-wide lock.
|
|
2885
3037
|
const before = JSON.stringify(state);
|
|
3038
|
+
if (state.version === 1) {
|
|
3039
|
+
// Keep v1 writable until a live owner commits its response and cooldown.
|
|
3040
|
+
// Returning the shared transient reason preserves cached identity/leases.
|
|
3041
|
+
if (Object.values(state.hosts).some((transport) =>
|
|
3042
|
+
transport.permit && !pidIsDead(transport.permit.pid, kill))) return { ok: false, reason: "busy" };
|
|
3043
|
+
for (const transport of Object.values(state.hosts)) {
|
|
3044
|
+
transport.permits = transport.permit
|
|
3045
|
+
? [{ ...transport.permit, kind: "legacy", bootId: coordinationBootId() }] : [];
|
|
3046
|
+
delete transport.permit;
|
|
3047
|
+
}
|
|
3048
|
+
state.version = IDENTITY_VERSION;
|
|
3049
|
+
}
|
|
3050
|
+
|
|
2886
3051
|
for (const transport of Object.values(state.hosts)) {
|
|
2887
3052
|
transport.waiters = transport.waiters.filter((waiter) => waiter.deadline > now && !pidIsDead(waiter.pid, kill));
|
|
2888
3053
|
}
|
|
@@ -2913,9 +3078,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
|
|
|
2913
3078
|
// stops the entry from being retained for the life of the install.
|
|
2914
3079
|
: budget?.resetMs > attempt.startedAt + IDENTITY_UNCERTAIN_MAX_MS;
|
|
2915
3080
|
if (!attempt.accounted && !resetCovered) return { ok: true, value: false };
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
3081
|
+
// Retiring registry attempt history never retires quota debt. A reset
|
|
3082
|
+
// proves a counter window changed, not that a suspended child cannot
|
|
3083
|
+
// dispatch after it resumes.
|
|
3084
|
+
return { ok: true, value: true };
|
|
2919
3085
|
});
|
|
2920
3086
|
if (!retired.ok) return retired;
|
|
2921
3087
|
if (retired.value) delete state.attempts[id];
|
|
@@ -3010,9 +3176,10 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
|
|
|
3010
3176
|
return withIdentityRegistry(root, (state) => {
|
|
3011
3177
|
const migration = inspectLegacyMigration(root, state, now);
|
|
3012
3178
|
if (!migration.ok) return migration;
|
|
3013
|
-
const transport = state.hosts[host] ??=
|
|
3014
|
-
|
|
3015
|
-
if (transport.
|
|
3179
|
+
const transport = state.hosts[host] ??= emptyIdentityTransport();
|
|
3180
|
+
transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, now, kill));
|
|
3181
|
+
if (transport.throttle.paused) return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
|
|
3182
|
+
if (transport.permits.some((permit) => permit.kind !== "legacy") || transport.waiters.length) return { ok: false, reason: "identity-busy", retryAt: now + 1000 };
|
|
3016
3183
|
const attempts = Object.values(state.attempts);
|
|
3017
3184
|
const recent = attempts.filter((attempt) => attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
|
|
3018
3185
|
const own = recent.filter((attempt) => attempt.credentialKey === credentialKey && attempt.resource === resource && attempt.exceptional);
|
|
@@ -3027,7 +3194,7 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
|
|
|
3027
3194
|
const nonce = governorId();
|
|
3028
3195
|
state.attempts[id] = { credentialKey, host, resource, startedAt: now, retryAt: now + attemptRetryDelayMs(own.length),
|
|
3029
3196
|
ownerPid: process.pid, nonce, status: "started", quotaKey: null, resetMs: null, accounted: false, imported: false, exceptional: true };
|
|
3030
|
-
transport.
|
|
3197
|
+
transport.permits.push({ pid: process.pid, nonce, startedAt: now, kind: "bootstrap", bootId: null });
|
|
3031
3198
|
transport.lastStartedAt = now;
|
|
3032
3199
|
return { ok: true, value: { id, nonce } };
|
|
3033
3200
|
}, { now, kill });
|
|
@@ -3045,32 +3212,31 @@ function importIdentityDebts(root, state, credentialKey, identity, now) {
|
|
|
3045
3212
|
return withGovernorLock(scope, () => {
|
|
3046
3213
|
const loaded = readIdentityQuotaState(state, scope, now);
|
|
3047
3214
|
if (!loaded.ok) return loaded;
|
|
3215
|
+
for (const id of Object.keys(loaded.value.importMarkers)) {
|
|
3216
|
+
const attempt = state.attempts[id];
|
|
3217
|
+
if (!attempt || attempt.imported && attempt.quotaKey === identity.quotaKey) {
|
|
3218
|
+
delete loaded.value.importMarkers[id];
|
|
3219
|
+
}
|
|
3220
|
+
}
|
|
3048
3221
|
const transferred = [];
|
|
3049
3222
|
for (const [id, attempt] of Object.entries(state.attempts)) {
|
|
3050
3223
|
if (attempt.credentialKey !== credentialKey || attempt.imported) continue;
|
|
3051
|
-
const
|
|
3052
|
-
loaded.value.reservations[reservationId] ??= {
|
|
3224
|
+
const legacyControl = {
|
|
3053
3225
|
leaseId: id, intentId: id, costs: { core: 0, graphql: 0, [attempt.resource]: 1 }, actualCosts: null,
|
|
3054
3226
|
accountedCosts: { core: 0, graphql: 0, [attempt.resource]: attempt.accounted ? 1 : 0 },
|
|
3055
3227
|
notBefore: attempt.startedAt, status: "started", epochs: { core: null, graphql: null },
|
|
3056
3228
|
startedAt: attempt.startedAt, completedAt: null, outcome: null,
|
|
3057
3229
|
};
|
|
3230
|
+
if (!loaded.value.importMarkers[id] &&
|
|
3231
|
+
!transferReceiptToDebt(loaded.value, legacyControl, now, id)) {
|
|
3232
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
3233
|
+
}
|
|
3234
|
+
if (Object.keys(loaded.value.importMarkers).length >= IDENTITY_MAX_ATTEMPTS &&
|
|
3235
|
+
!loaded.value.importMarkers[id]) return { ok: false, reason: "identity-capacity" };
|
|
3236
|
+
loaded.value.importMarkers[id] = true;
|
|
3058
3237
|
transferred.push(id);
|
|
3059
3238
|
}
|
|
3060
|
-
|
|
3061
|
-
// that records the mapping commits. A crash in between leaves a charged
|
|
3062
|
-
// receipt here whose attempt is gone from the registry, and nothing else
|
|
3063
|
-
// ever looks at it again -- it presents only as a permanently smaller
|
|
3064
|
-
// budget. Re-bootstrapping the same credential is exactly when it can be
|
|
3065
|
-
// recognised, and a full window since it started proves a reset covered it.
|
|
3066
|
-
for (const key of Object.keys(loaded.value.reservations)) {
|
|
3067
|
-
const attemptId = /^reservation:(.+)$/.exec(key)?.[1];
|
|
3068
|
-
if (!attemptId || state.attempts[attemptId]) continue;
|
|
3069
|
-
const reservation = loaded.value.reservations[key];
|
|
3070
|
-
if (reservation.startedAt + IDENTITY_UNCERTAIN_MAX_MS > now) continue;
|
|
3071
|
-
delete loaded.value.reservations[key];
|
|
3072
|
-
}
|
|
3073
|
-
const written = writeGovernorState(scope.path, loaded.value);
|
|
3239
|
+
const written = writeGovernorQuotaState(scope.path, loaded.value, now);
|
|
3074
3240
|
if (!written.ok) return written;
|
|
3075
3241
|
// Marked transferred only once the receipts are durable. withIdentityRegistry
|
|
3076
3242
|
// persists the registry whatever this returns, so setting these before the
|
|
@@ -3090,7 +3256,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
|
|
|
3090
3256
|
const attempt = state.attempts[id];
|
|
3091
3257
|
if (!attempt || attempt.nonce !== nonce || attempt.credentialKey !== credentialKey) return { ok: false, reason: "stale" };
|
|
3092
3258
|
const transport = state.hosts[attempt.host];
|
|
3093
|
-
|
|
3259
|
+
transport.permits = transport.permits.filter((permit) => permit.nonce !== nonce);
|
|
3094
3260
|
attempt.status = response ? "finished" : "uncertain";
|
|
3095
3261
|
const parsed = response?.body;
|
|
3096
3262
|
const evidence = response?.rateLimit;
|
|
@@ -3128,7 +3294,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
|
|
|
3128
3294
|
ledger.value.epochs.core = observation.budget.epoch;
|
|
3129
3295
|
ledger.value.observers.core = { etag: typeof response.etag === "string" ? response.etag : null, outcome: "healthy", at: now, nextAt: now + BUDGET_PROBE_MS };
|
|
3130
3296
|
}
|
|
3131
|
-
return
|
|
3297
|
+
return writeGovernorQuotaState(scope.path, ledger.value, now);
|
|
3132
3298
|
});
|
|
3133
3299
|
if (!seeded.ok) return seeded;
|
|
3134
3300
|
}
|
|
@@ -3267,35 +3433,53 @@ function startIdentityControl(coordinator, now = Date.now()) {
|
|
|
3267
3433
|
const exceptional = !budget || now >= budget.resetMs + BUDGET_RESET_GRACE_MS;
|
|
3268
3434
|
if (budget && now >= budget.resetMs && !exceptional) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
|
|
3269
3435
|
if (!exceptional) {
|
|
3270
|
-
const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now),
|
|
3436
|
+
const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now),
|
|
3437
|
+
state.debt.core.unresolvedUnits + state.debt.core.quiescentUnits +
|
|
3438
|
+
reservationCost(state.controlReceipts.core, "core", state.leases, now));
|
|
3271
3439
|
if (budget.remaining - resourceReserve(budget.limit) - reserved < 1) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
|
|
3272
3440
|
}
|
|
3273
3441
|
const recent = Object.values(registry.attempts).filter((attempt) => attempt.credentialKey === identity.credentialKey && attempt.resource === "core" && attempt.exceptional && attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
|
|
3274
3442
|
const retryAt = Math.max(0, ...recent.map((attempt) => attempt.retryAt), recent.length >= IDENTITY_ATTEMPT_LIMIT ? Math.min(...recent.map((attempt) => attempt.startedAt)) + IDENTITY_ATTEMPT_WINDOW_MS : 0);
|
|
3275
3443
|
if (exceptional && retryAt > now) return { ok: false, reason: "identity-backoff", retryAt };
|
|
3276
|
-
if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS
|
|
3444
|
+
if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS) return { ok: false, reason: "identity-capacity" };
|
|
3445
|
+
const priorControl = state.controlReceipts.core;
|
|
3446
|
+
if (priorControl) {
|
|
3447
|
+
if (!retireControlReceiptToDebt(state, "core", now)) return { ok: false, reason: "accounting-overflow" };
|
|
3448
|
+
}
|
|
3277
3449
|
const id = governorId();
|
|
3278
3450
|
registry.attempts[id] = { credentialKey: identity.credentialKey, host: identity.host, resource: "core", startedAt: now,
|
|
3279
3451
|
retryAt: exceptional ? now + attemptRetryDelayMs(recent.length) : now, ownerPid: process.pid, nonce: governorId(), status: "started",
|
|
3280
3452
|
quotaKey: identity.quotaKey, resetMs: budget?.resetMs ?? null, accounted: false, imported: true, exceptional };
|
|
3281
|
-
state.
|
|
3453
|
+
state.controlReceipts.core = { leaseId: id, intentId: id, costs: { core: 1, graphql: 0 }, actualCosts: null,
|
|
3282
3454
|
accountedCosts: { core: 0, graphql: 0 }, notBefore: now, status: "started", epochs: { core: budget?.epoch ?? null, graphql: null },
|
|
3283
|
-
startedAt: now, completedAt: null, outcome: null
|
|
3284
|
-
|
|
3285
|
-
|
|
3455
|
+
startedAt: now, completedAt: null, outcome: null,
|
|
3456
|
+
receipt: {
|
|
3457
|
+
scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
|
|
3458
|
+
ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
|
|
3459
|
+
deadline: now + ACQUISITION_STARTED_DEADLINE_MS,
|
|
3460
|
+
allowance: { core: 1, graphql: 0 }, dispatches: [],
|
|
3461
|
+
} };
|
|
3462
|
+
const written = writeGovernorQuotaState(scope.path, state, now);
|
|
3463
|
+
return written.ok ? { ok: true, value: { id, identity,
|
|
3464
|
+
receiptCapability: { reservationId: `control:core`, controlResource: "core",
|
|
3465
|
+
leaseId: id, scopeHash: scope.hash,
|
|
3466
|
+
ownerNonce: state.controlReceipts.core.receipt.ownerNonce,
|
|
3467
|
+
generation: state.controlReceipts.core.receipt.generation,
|
|
3468
|
+
deadline: state.controlReceipts.core.receipt.deadline } } } : written;
|
|
3286
3469
|
});
|
|
3287
3470
|
}, { now });
|
|
3288
3471
|
}
|
|
3289
3472
|
|
|
3290
|
-
function settleIdentityControl(coordinator, control, stdout, now = Date.now()
|
|
3473
|
+
function settleIdentityControl(coordinator, control, stdout, now = Date.now(),
|
|
3474
|
+
{ neverStarted = false } = {}) {
|
|
3291
3475
|
return withIdentityRegistry(coordinator.root, (registry) => {
|
|
3292
3476
|
const attempt = registry.attempts[control.id];
|
|
3293
3477
|
if (!attempt) return { ok: false, reason: "stale" };
|
|
3294
3478
|
const parsed = parseGhApiResponse(stdout ?? "");
|
|
3295
3479
|
const evidence = pickRateLimit(parsed.headers);
|
|
3296
3480
|
const proven = [200, 304].includes(parsed.status) && evidence?.resource === "core";
|
|
3297
|
-
attempt.status = proven ? "finished" : "uncertain";
|
|
3298
|
-
attempt.accounted = proven;
|
|
3481
|
+
attempt.status = proven || neverStarted ? "finished" : "uncertain";
|
|
3482
|
+
attempt.accounted = proven || neverStarted;
|
|
3299
3483
|
if (evidence?.resource === "core") {
|
|
3300
3484
|
attempt.resetMs = evidence.resetMs;
|
|
3301
3485
|
if (evidence.remaining === 0) attempt.retryAt = Math.max(attempt.retryAt, evidence.resetMs + BUDGET_RESET_GRACE_MS);
|
|
@@ -3304,64 +3488,93 @@ function settleIdentityControl(coordinator, control, stdout, now = Date.now()) {
|
|
|
3304
3488
|
return withGovernorLock(scope, () => {
|
|
3305
3489
|
const loaded = readIdentityQuotaState(registry, scope, now);
|
|
3306
3490
|
if (!loaded.ok) return loaded;
|
|
3307
|
-
const reservation = loaded.value.
|
|
3491
|
+
const reservation = loaded.value.controlReceipts.core?.intentId === control.id
|
|
3492
|
+
? loaded.value.controlReceipts.core : null;
|
|
3308
3493
|
if (!reservation) return { ok: false, reason: "stale" };
|
|
3309
|
-
if (proven) {
|
|
3310
|
-
const cost = parsed.status === 304 ? 0 : 1;
|
|
3494
|
+
if (proven || neverStarted) {
|
|
3495
|
+
const cost = neverStarted || parsed.status === 304 ? 0 : 1;
|
|
3311
3496
|
reservation.status = "completed";
|
|
3312
3497
|
reservation.completedAt = now;
|
|
3313
3498
|
reservation.outcome = "measured-success";
|
|
3314
3499
|
reservation.actualCosts = { core: cost, graphql: 0 };
|
|
3315
3500
|
reservation.accountedCosts = { core: cost, graphql: 0 };
|
|
3501
|
+
loaded.value.controlReceipts.core = null;
|
|
3502
|
+
} else if (!retireControlReceiptToDebt(loaded.value, "core", now)) {
|
|
3503
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
3316
3504
|
}
|
|
3317
|
-
return
|
|
3505
|
+
return writeGovernorQuotaState(scope.path, loaded.value, now);
|
|
3318
3506
|
});
|
|
3319
3507
|
}, { now });
|
|
3320
3508
|
}
|
|
3321
3509
|
|
|
3510
|
+
const HTTP_PERMIT_FAILURE_HOLDS = Object.freeze({
|
|
3511
|
+
"transport-busy": "coordination",
|
|
3512
|
+
"throttle-paused": "secondary",
|
|
3513
|
+
});
|
|
3514
|
+
|
|
3515
|
+
function httpPermitFailure(reason) {
|
|
3516
|
+
const error = new Error(identityCoordinationMessage(reason));
|
|
3517
|
+
if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, reason)) error.coordinationReason = reason;
|
|
3518
|
+
return error;
|
|
3519
|
+
}
|
|
3520
|
+
|
|
3322
3521
|
async function acquireIdentityHttpPermit(coordinator, {
|
|
3323
3522
|
signal,
|
|
3324
3523
|
now = Date.now,
|
|
3325
3524
|
wait = (delay) => new Promise((resolve) => setTimeout(resolve, delay)),
|
|
3525
|
+
operationDeadline,
|
|
3326
3526
|
} = {}) {
|
|
3327
3527
|
const flushed = await coordinator.flushCompletions?.();
|
|
3328
|
-
if (flushed && !flushed.ok) throw
|
|
3528
|
+
if (flushed && !flushed.ok) throw httpPermitFailure(flushed.reason);
|
|
3329
3529
|
const identity = coordinator.current();
|
|
3330
3530
|
if (!identity) throw new Error("Verified identity unavailable");
|
|
3331
3531
|
const queuedAt = now();
|
|
3332
|
-
const deadline =
|
|
3532
|
+
const deadline = Number.isFinite(operationDeadline)
|
|
3533
|
+
? Math.min(operationDeadline, queuedAt + ACQUISITION_STARTED_DEADLINE_MS)
|
|
3534
|
+
: queuedAt + GH_TIMEOUT_MS;
|
|
3333
3535
|
const nonce = governorId();
|
|
3334
3536
|
let acquired = false;
|
|
3335
3537
|
try {
|
|
3336
3538
|
for (;;) {
|
|
3337
3539
|
if (signal?.aborted) throw new Error("Request cancelled");
|
|
3338
3540
|
if (coordinator.current()?.accessKey !== identity.accessKey) throw new Error("Credential changed before request start");
|
|
3339
|
-
if (now() >= deadline) throw
|
|
3541
|
+
if (now() >= deadline) throw httpPermitFailure("transport-busy");
|
|
3340
3542
|
const result = withIdentityRegistry(coordinator.root, (state) => {
|
|
3341
3543
|
const migration = inspectLegacyMigration(coordinator.root, state, now());
|
|
3342
3544
|
if (!migration.ok) return migration;
|
|
3343
|
-
|
|
3344
|
-
|
|
3545
|
+
const permitAt = now();
|
|
3546
|
+
if (permitAt >= deadline) return { ok: false, reason: "transport-busy", retryAt: deadline };
|
|
3547
|
+
const transport = state.hosts[identity.host] ??= emptyIdentityTransport();
|
|
3345
3548
|
// Five consecutive throttles means the ladder has stopped being useful.
|
|
3346
3549
|
// Waiting out the deadline is what produced the last one, so the pause
|
|
3347
3550
|
// holds until a person retries or a primary reset opens a recovery.
|
|
3348
3551
|
if (transport.throttle?.paused) {
|
|
3349
3552
|
return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
|
|
3350
3553
|
}
|
|
3351
|
-
|
|
3554
|
+
transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, permitAt));
|
|
3352
3555
|
// Admission can happen during the mandatory start gap. Persist call
|
|
3353
3556
|
// order once so later polling cannot overtake an earlier caller.
|
|
3354
|
-
|
|
3557
|
+
let waiter = transport.waiters.find((item) => item.nonce === nonce);
|
|
3558
|
+
if (!waiter) {
|
|
3355
3559
|
if (transport.waiters.length >= HTTP_MAX_WAITERS) return { ok: false, reason: "identity-capacity" };
|
|
3356
|
-
|
|
3560
|
+
waiter = { pid: process.pid, nonce, queuedAt: permitAt, deadline: Math.min(deadline, permitAt + GH_TIMEOUT_MS) };
|
|
3561
|
+
transport.waiters.push(waiter);
|
|
3562
|
+
} else if (waiter.deadline - permitAt <= GH_TIMEOUT_MS / 2 && waiter.deadline < deadline) {
|
|
3563
|
+
// Keep the existing registry's short waiter lease and FIFO position.
|
|
3564
|
+
// Renewing this lease cannot extend the admitted operation deadline
|
|
3565
|
+
// above, or the separate timeout of a child that actually starts.
|
|
3566
|
+
waiter.queuedAt = permitAt;
|
|
3567
|
+
waiter.deadline = Math.min(deadline, waiter.queuedAt + GH_TIMEOUT_MS);
|
|
3357
3568
|
}
|
|
3358
3569
|
const retryAt = Math.max(transport.cooldownUntil, transport.lastStartedAt + HTTP_START_GAP_MS);
|
|
3359
|
-
if (transport.
|
|
3360
|
-
|
|
3570
|
+
if (transport.permits.length >= HTTP_MAX_PERMITS ||
|
|
3571
|
+
transport.permits.some((permit) => permit.kind === "bootstrap") ||
|
|
3572
|
+
transport.waiters[0].nonce !== nonce || retryAt > permitAt) {
|
|
3573
|
+
return { ok: false, reason: "transport-busy", retryAt: Math.max(retryAt, permitAt + 50) };
|
|
3361
3574
|
}
|
|
3362
3575
|
transport.waiters.shift();
|
|
3363
|
-
transport.
|
|
3364
|
-
transport.lastStartedAt =
|
|
3576
|
+
transport.permits.push({ pid: process.pid, nonce, startedAt: permitAt, kind: "request", bootId: null });
|
|
3577
|
+
transport.lastStartedAt = permitAt;
|
|
3365
3578
|
return { ok: true, value: { nonce, host: identity.host } };
|
|
3366
3579
|
}, { now: now() });
|
|
3367
3580
|
if (result.ok) { acquired = true; return result.value; }
|
|
@@ -3370,7 +3583,7 @@ async function acquireIdentityHttpPermit(coordinator, {
|
|
|
3370
3583
|
// retryAt, so give the rest the poll interval the loop already uses.
|
|
3371
3584
|
const retryAt = result.retryAt ?? now() + 50;
|
|
3372
3585
|
if ((result.reason !== "transport-busy" && !retryableCoordination(result.reason)) ||
|
|
3373
|
-
now() >= deadline || retryAt >= deadline) throw
|
|
3586
|
+
now() >= deadline || retryAt >= deadline) throw httpPermitFailure(result.reason);
|
|
3374
3587
|
await wait(Math.max(1, Math.min(50, retryAt - now())));
|
|
3375
3588
|
}
|
|
3376
3589
|
} finally {
|
|
@@ -3454,6 +3667,10 @@ function transportCooldownDeadline({ retryAfter, status, secondary, at, attempts
|
|
|
3454
3667
|
return status === 429 || secondary === true ? at + throttleLadderMs(attempts) : null;
|
|
3455
3668
|
}
|
|
3456
3669
|
|
|
3670
|
+
function emptyIdentityTransport() {
|
|
3671
|
+
return { lastStartedAt: 0, cooldownUntil: 0, permits: [], waiters: [], throttle: emptyTransportThrottle() };
|
|
3672
|
+
}
|
|
3673
|
+
|
|
3457
3674
|
function emptyTransportThrottle() {
|
|
3458
3675
|
return { attempts: 0, lastAt: 0, paused: false };
|
|
3459
3676
|
}
|
|
@@ -3481,10 +3698,21 @@ function clearTransportThrottle(transport) {
|
|
|
3481
3698
|
}
|
|
3482
3699
|
}
|
|
3483
3700
|
|
|
3701
|
+
// Completion retries keep their original response time and become no-ops
|
|
3702
|
+
// after a successful durable write. Weak keys do not accumulate tombstones.
|
|
3703
|
+
const identityPermitCompletions = new WeakMap();
|
|
3484
3704
|
function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date.now()) {
|
|
3485
|
-
|
|
3705
|
+
let completion = identityPermitCompletions.get(permit);
|
|
3706
|
+
if (!completion) {
|
|
3707
|
+
completion = { at: now, done: false };
|
|
3708
|
+
identityPermitCompletions.set(permit, completion);
|
|
3709
|
+
}
|
|
3710
|
+
if (completion.done) return { ok: true };
|
|
3711
|
+
now = completion.at;
|
|
3712
|
+
const result = withIdentityRegistry(coordinator.root, (state) => {
|
|
3486
3713
|
const transport = state.hosts[permit.host];
|
|
3487
|
-
|
|
3714
|
+
const held = transport?.permits.find((item) => item.nonce === permit.nonce);
|
|
3715
|
+
if (transport) transport.permits = transport.permits.filter((item) => item.nonce !== permit.nonce);
|
|
3488
3716
|
if (transport && error) {
|
|
3489
3717
|
const response = parseGhApiResponse(error.stdout ?? "");
|
|
3490
3718
|
const verdict = classifyThrottle({
|
|
@@ -3503,11 +3731,14 @@ function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date
|
|
|
3503
3731
|
at: now,
|
|
3504
3732
|
});
|
|
3505
3733
|
}
|
|
3506
|
-
} else if (transport && !error
|
|
3734
|
+
} else if (transport && !error && held && held.startedAt > transport.throttle.lastAt &&
|
|
3735
|
+
held.startedAt >= transport.cooldownUntil) {
|
|
3507
3736
|
clearTransportThrottle(transport);
|
|
3508
3737
|
}
|
|
3509
3738
|
return { ok: true };
|
|
3510
3739
|
});
|
|
3740
|
+
if (result.ok) completion.done = true;
|
|
3741
|
+
return result;
|
|
3511
3742
|
}
|
|
3512
3743
|
|
|
3513
3744
|
// The explicit retry a paused transport waits for. Clears the ladder and the
|
|
@@ -3558,11 +3789,11 @@ const GOVERNOR_SCOPE_VERSION = 1;
|
|
|
3558
3789
|
// 5: scheduling fairness is persisted, because the starvation it prevents
|
|
3559
3790
|
// happens *between* planning passes -- each manual refresh is its own pass, so
|
|
3560
3791
|
// a counter that lives only inside one cannot see a run of them.
|
|
3561
|
-
const GOVERNOR_STATE_VERSION =
|
|
3792
|
+
const GOVERNOR_STATE_VERSION = 7;
|
|
3562
3793
|
// Readable-as-evidence, never written: the shapes a still-running older pane
|
|
3563
3794
|
// holds. Recognising that such a pane owns a live lease is what the restart
|
|
3564
3795
|
// boundary depends on.
|
|
3565
|
-
const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 5, 4, 3, 2];
|
|
3796
|
+
const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 6, 5, 4, 3, 2];
|
|
3566
3797
|
// The first version whose file is already in the per-resource observer shape.
|
|
3567
3798
|
// Anything below it is adapted before it can be read; see readLegacyGovernorState.
|
|
3568
3799
|
const GOVERNOR_OBSERVER_SPLIT_VERSION = 4;
|
|
@@ -3582,6 +3813,8 @@ const MANUAL_GRANT_STREAK_LIMIT = 3;
|
|
|
3582
3813
|
const GOVERNOR_MAX_LEASES = 128;
|
|
3583
3814
|
const GOVERNOR_MAX_INTENTS = 512;
|
|
3584
3815
|
const GOVERNOR_MAX_RESERVATIONS = 512;
|
|
3816
|
+
const GOVERNOR_MAX_DEBT_GROUPS = 128;
|
|
3817
|
+
const GOVERNOR_MAX_LEDGER_BYTES = 2 * 1024 * 1024;
|
|
3585
3818
|
const GOVERNOR_LOCK_WAIT_MS = 250;
|
|
3586
3819
|
// How old an unreadable lock record may be before it counts as abandoned. A
|
|
3587
3820
|
// creator owns the path from open("wx") until its owner record is written, a
|
|
@@ -3697,6 +3930,7 @@ function currentGovernorScope(scope) {
|
|
|
3697
3930
|
function emptyGovernorState() {
|
|
3698
3931
|
return {
|
|
3699
3932
|
version: GOVERNOR_STATE_VERSION,
|
|
3933
|
+
revision: 0,
|
|
3700
3934
|
epochs: { core: null, graphql: null },
|
|
3701
3935
|
budgets: {},
|
|
3702
3936
|
fairness: { manualStreak: 0 },
|
|
@@ -3712,6 +3946,14 @@ function emptyGovernorState() {
|
|
|
3712
3946
|
leases: {},
|
|
3713
3947
|
intents: {},
|
|
3714
3948
|
reservations: {},
|
|
3949
|
+
controlReceipts: { core: null, graphql: null },
|
|
3950
|
+
debt: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, {
|
|
3951
|
+
unresolvedUnits: 0, unresolvedCount: 0, quiescentUnits: 0, quiescentCount: 0,
|
|
3952
|
+
revision: 0,
|
|
3953
|
+
}])),
|
|
3954
|
+
debtGroups: {},
|
|
3955
|
+
ownerGenerations: {},
|
|
3956
|
+
importMarkers: {},
|
|
3715
3957
|
manualProbe: null,
|
|
3716
3958
|
};
|
|
3717
3959
|
}
|
|
@@ -3804,10 +4046,11 @@ function normalizeGovernorIntent(raw, nowMs) {
|
|
|
3804
4046
|
return { ...raw, costs };
|
|
3805
4047
|
}
|
|
3806
4048
|
|
|
3807
|
-
function normalizeGovernorReservation(raw, nowMs) {
|
|
4049
|
+
function normalizeGovernorReservation(raw, nowMs, { legacy = false } = {}) {
|
|
3808
4050
|
if (!exactKeys(raw, [
|
|
3809
4051
|
"leaseId", "intentId", "costs", "actualCosts", "accountedCosts", "notBefore", "status", "epochs",
|
|
3810
4052
|
"startedAt", "completedAt", "outcome",
|
|
4053
|
+
...(!legacy ? ["receipt"] : []),
|
|
3811
4054
|
])) return null;
|
|
3812
4055
|
const costs = exactResourceCosts(raw.costs);
|
|
3813
4056
|
const actualCosts = raw.actualCosts === null ? null : exactResourceCosts(raw.actualCosts);
|
|
@@ -3829,13 +4072,46 @@ function normalizeGovernorReservation(raw, nowMs) {
|
|
|
3829
4072
|
) return null;
|
|
3830
4073
|
if (actualCosts && RATE_RESOURCES.some((resource) => actualCosts[resource] > costs[resource])) return null;
|
|
3831
4074
|
if (RATE_RESOURCES.some((resource) => accountedCosts[resource] > (actualCosts?.[resource] ?? costs[resource]))) return null;
|
|
4075
|
+
if (!legacy && raw.receipt !== null) {
|
|
4076
|
+
const receipt = raw.receipt;
|
|
4077
|
+
if (!exactKeys(receipt, [
|
|
4078
|
+
"scopeHash", "ownerNonce", "generation", "ownerPid", "ownerBirth", "bootId",
|
|
4079
|
+
"deadline", "allowance", "dispatches",
|
|
4080
|
+
]) || typeof receipt.scopeHash !== "string" || !/^[0-9a-f]{64}$/.test(receipt.scopeHash) ||
|
|
4081
|
+
!validGovernorId(receipt.ownerNonce) || !validGovernorId(receipt.generation) ||
|
|
4082
|
+
!Number.isSafeInteger(receipt.ownerPid) || receipt.ownerPid < 1 ||
|
|
4083
|
+
receipt.ownerBirth !== null && (typeof receipt.ownerBirth !== "string" || receipt.ownerBirth.length > 128) ||
|
|
4084
|
+
receipt.bootId !== null && (typeof receipt.bootId !== "string" || receipt.bootId.length > 128) ||
|
|
4085
|
+
finiteTimestamp(receipt.deadline, nowMs) === undefined ||
|
|
4086
|
+
!exactKeys(receipt.allowance, RATE_RESOURCES) ||
|
|
4087
|
+
RATE_RESOURCES.some((resource) => receipt.allowance[resource] !== costs[resource]) ||
|
|
4088
|
+
!Array.isArray(receipt.dispatches) || receipt.dispatches.length > 32) return null;
|
|
4089
|
+
const issued = { core: 0, graphql: 0 };
|
|
4090
|
+
for (const [index, dispatch] of receipt.dispatches.entries()) {
|
|
4091
|
+
if (!exactKeys(dispatch, ["sequence", "operation", "costs", "issuedAt", "terminalAt", "childPid", "childBirth", "neverStarted"]) ||
|
|
4092
|
+
dispatch.sequence !== index + 1 || !operationCost(dispatch.operation) ||
|
|
4093
|
+
!exactKeys(dispatch.costs, RATE_RESOURCES) ||
|
|
4094
|
+
RATE_RESOURCES.some((resource) => dispatch.costs[resource] !== operationCost(dispatch.operation)[resource]) ||
|
|
4095
|
+
finiteTimestamp(dispatch.issuedAt, nowMs) === undefined ||
|
|
4096
|
+
finiteTimestamp(dispatch.terminalAt, nowMs, { nullable: true }) === undefined ||
|
|
4097
|
+
dispatch.childPid !== null && (!Number.isSafeInteger(dispatch.childPid) || dispatch.childPid < 1) ||
|
|
4098
|
+
dispatch.childBirth !== null && (typeof dispatch.childBirth !== "string" || dispatch.childBirth.length > 128) ||
|
|
4099
|
+
typeof dispatch.neverStarted !== "boolean" ||
|
|
4100
|
+
dispatch.neverStarted && (dispatch.childPid !== null || dispatch.terminalAt === null)) return null;
|
|
4101
|
+
for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
|
|
4102
|
+
}
|
|
4103
|
+
if (RATE_RESOURCES.some((resource) => !Number.isSafeInteger(issued[resource]) ||
|
|
4104
|
+
issued[resource] > receipt.allowance[resource])) return null;
|
|
4105
|
+
} else if (!legacy && raw.status === "started") {
|
|
4106
|
+
return null;
|
|
4107
|
+
}
|
|
3832
4108
|
return { ...raw, costs, actualCosts, accountedCosts, epochs };
|
|
3833
4109
|
}
|
|
3834
4110
|
|
|
3835
4111
|
function normalizeResourceObserver(raw, nowMs) {
|
|
3836
4112
|
if (!exactKeys(raw, ["etag", "outcome", "at", "nextAt"])) return null;
|
|
3837
4113
|
if (
|
|
3838
|
-
raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0) ||
|
|
4114
|
+
raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0 || raw.etag.length > 512) ||
|
|
3839
4115
|
!GOVERNOR_PROBE_STATUSES.has(raw.outcome) ||
|
|
3840
4116
|
finiteTimestamp(raw.at, nowMs) === undefined || raw.at > nowMs ||
|
|
3841
4117
|
finiteTimestamp(raw.nextAt, nowMs) === undefined
|
|
@@ -3843,10 +4119,11 @@ function normalizeResourceObserver(raw, nowMs) {
|
|
|
3843
4119
|
return { ...raw };
|
|
3844
4120
|
}
|
|
3845
4121
|
|
|
3846
|
-
function normalizeProbeClaim(raw, nowMs) {
|
|
4122
|
+
function normalizeProbeClaim(raw, nowMs, { legacy = false } = {}) {
|
|
3847
4123
|
if (raw === null) return null;
|
|
3848
4124
|
if (!exactKeys(raw, [
|
|
3849
4125
|
"ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds",
|
|
4126
|
+
...(!legacy ? ["debtSnapshot"] : []),
|
|
3850
4127
|
])) {
|
|
3851
4128
|
return undefined;
|
|
3852
4129
|
}
|
|
@@ -3858,6 +4135,12 @@ function normalizeProbeClaim(raw, nowMs) {
|
|
|
3858
4135
|
!Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
|
|
3859
4136
|
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
|
|
3860
4137
|
) return undefined;
|
|
4138
|
+
const snapshot = raw.debtSnapshot;
|
|
4139
|
+
if (!legacy && (!exactKeys(snapshot, ["units", "count", "barrierAt", "revision"]) ||
|
|
4140
|
+
!Number.isSafeInteger(snapshot.units) || snapshot.units < 0 ||
|
|
4141
|
+
!Number.isSafeInteger(snapshot.count) || snapshot.count < 0 ||
|
|
4142
|
+
finiteTimestamp(snapshot.barrierAt, nowMs) === undefined ||
|
|
4143
|
+
!Number.isSafeInteger(snapshot.revision) || snapshot.revision < 0)) return undefined;
|
|
3861
4144
|
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
|
|
3862
4145
|
}
|
|
3863
4146
|
|
|
@@ -3878,9 +4161,11 @@ function normalizeManualProbe(raw, nowMs) {
|
|
|
3878
4161
|
// never publishes what it reads. Every other caller takes the default, so a
|
|
3879
4162
|
// version this build does not write is unreadable and fails closed.
|
|
3880
4163
|
function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVERNOR_STATE_VERSION } = {}) {
|
|
4164
|
+
const legacyShape = acceptVersion < 7;
|
|
3881
4165
|
if (!exactKeys(raw, [
|
|
3882
4166
|
"version", "epochs", "budgets", "fairness", "observers", "probeClaims", "leases",
|
|
3883
4167
|
"intents", "reservations", "manualProbe",
|
|
4168
|
+
...(!legacyShape ? ["revision", "controlReceipts", "debt", "debtGroups", "ownerGenerations", "importMarkers"] : []),
|
|
3884
4169
|
]) || raw.version !== acceptVersion) return null;
|
|
3885
4170
|
if (!exactKeys(raw.fairness, ["manualStreak"]) ||
|
|
3886
4171
|
!Number.isSafeInteger(raw.fairness.manualStreak) || raw.fairness.manualStreak < 0) return null;
|
|
@@ -3894,6 +4179,75 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3894
4179
|
) return null;
|
|
3895
4180
|
|
|
3896
4181
|
const state = emptyGovernorState();
|
|
4182
|
+
state.version = acceptVersion;
|
|
4183
|
+
if (legacyShape) {
|
|
4184
|
+
delete state.revision;
|
|
4185
|
+
delete state.controlReceipts;
|
|
4186
|
+
delete state.debt;
|
|
4187
|
+
delete state.debtGroups;
|
|
4188
|
+
delete state.ownerGenerations;
|
|
4189
|
+
delete state.importMarkers;
|
|
4190
|
+
} else {
|
|
4191
|
+
if (!Number.isSafeInteger(raw.revision) || raw.revision < 0) return null;
|
|
4192
|
+
state.revision = raw.revision;
|
|
4193
|
+
if (!exactKeys(raw.controlReceipts, RATE_RESOURCES) || !exactKeys(raw.debt, RATE_RESOURCES) ||
|
|
4194
|
+
!isRecord(raw.debtGroups) || Object.keys(raw.debtGroups).length > GOVERNOR_MAX_DEBT_GROUPS ||
|
|
4195
|
+
!isRecord(raw.ownerGenerations) ||
|
|
4196
|
+
Object.keys(raw.ownerGenerations).length > GOVERNOR_MAX_LEASES + GOVERNOR_MAX_RESERVATIONS ||
|
|
4197
|
+
!isRecord(raw.importMarkers) || Object.keys(raw.importMarkers).length > IDENTITY_MAX_ATTEMPTS) return null;
|
|
4198
|
+
for (const [id, marker] of Object.entries(raw.importMarkers)) {
|
|
4199
|
+
if (!validGovernorId(id) || marker !== true) return null;
|
|
4200
|
+
state.importMarkers[id] = true;
|
|
4201
|
+
}
|
|
4202
|
+
for (const [leaseId, generation] of Object.entries(raw.ownerGenerations)) {
|
|
4203
|
+
if (!validGovernorId(leaseId) || !validGovernorId(generation)) return null;
|
|
4204
|
+
state.ownerGenerations[leaseId] = generation;
|
|
4205
|
+
}
|
|
4206
|
+
for (const resource of RATE_RESOURCES) {
|
|
4207
|
+
const debt = raw.debt[resource];
|
|
4208
|
+
if (!exactKeys(debt, ["unresolvedUnits", "unresolvedCount", "quiescentUnits", "quiescentCount", "revision"]) ||
|
|
4209
|
+
Object.values(debt).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
|
|
4210
|
+
state.debt[resource] = { ...debt };
|
|
4211
|
+
// Control receipts are normalized independently from the 512 data slots.
|
|
4212
|
+
if (raw.controlReceipts[resource] !== null) {
|
|
4213
|
+
const control = normalizeGovernorReservation(raw.controlReceipts[resource], nowMs);
|
|
4214
|
+
if (!control) return null;
|
|
4215
|
+
state.controlReceipts[resource] = control;
|
|
4216
|
+
}
|
|
4217
|
+
}
|
|
4218
|
+
for (const [id, group] of Object.entries(raw.debtGroups)) {
|
|
4219
|
+
if (!validGovernorId(id) || !exactKeys(group, [
|
|
4220
|
+
"nonce", "ownerNonce", "ownerPid", "ownerBirth", "bootId", "unknownChild", "sealed",
|
|
4221
|
+
"quiescent", "units", "counts", "at", "barrierAt", "barrierRevisions",
|
|
4222
|
+
]) || !validGovernorId(group.nonce) ||
|
|
4223
|
+
group.ownerNonce !== null && !validGovernorId(group.ownerNonce) ||
|
|
4224
|
+
group.ownerPid !== null && (!Number.isSafeInteger(group.ownerPid) || group.ownerPid < 1) ||
|
|
4225
|
+
group.ownerBirth !== null && (typeof group.ownerBirth !== "string" || group.ownerBirth.length > 128) ||
|
|
4226
|
+
group.bootId !== null && (typeof group.bootId !== "string" || group.bootId.length > 128) ||
|
|
4227
|
+
typeof group.unknownChild !== "boolean" || typeof group.sealed !== "boolean" ||
|
|
4228
|
+
typeof group.quiescent !== "boolean" || !exactKeys(group.units, RATE_RESOURCES) ||
|
|
4229
|
+
!exactKeys(group.counts, RATE_RESOURCES) ||
|
|
4230
|
+
[...Object.values(group.units), ...Object.values(group.counts)].some((value) =>
|
|
4231
|
+
!Number.isSafeInteger(value) || value < 0) ||
|
|
4232
|
+
finiteTimestamp(group.at, nowMs) === undefined ||
|
|
4233
|
+
finiteTimestamp(group.barrierAt, nowMs, { nullable: true }) === undefined ||
|
|
4234
|
+
!exactKeys(group.barrierRevisions, RATE_RESOURCES) ||
|
|
4235
|
+
Object.values(group.barrierRevisions).some((value) =>
|
|
4236
|
+
!Number.isSafeInteger(value) || value < 0)) return null;
|
|
4237
|
+
state.debtGroups[id] = structuredClone(group);
|
|
4238
|
+
}
|
|
4239
|
+
for (const resource of RATE_RESOURCES) {
|
|
4240
|
+
const totals = { unresolvedUnits: 0, unresolvedCount: 0,
|
|
4241
|
+
quiescentUnits: 0, quiescentCount: 0 };
|
|
4242
|
+
for (const group of Object.values(state.debtGroups)) {
|
|
4243
|
+
const prefix = group.quiescent ? "quiescent" : "unresolved";
|
|
4244
|
+
totals[`${prefix}Units`] += group.units[resource];
|
|
4245
|
+
totals[`${prefix}Count`] += group.counts[resource];
|
|
4246
|
+
}
|
|
4247
|
+
if (Object.entries(totals).some(([field, amount]) =>
|
|
4248
|
+
!Number.isSafeInteger(amount) || amount !== state.debt[resource][field])) return null;
|
|
4249
|
+
}
|
|
4250
|
+
}
|
|
3897
4251
|
state.epochs = { ...raw.epochs };
|
|
3898
4252
|
state.fairness = { ...raw.fairness };
|
|
3899
4253
|
for (const resource of RATE_RESOURCES) {
|
|
@@ -3922,7 +4276,7 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3922
4276
|
if (Object.keys(state.intents).length > GOVERNOR_MAX_INTENTS) return null;
|
|
3923
4277
|
for (const [id, rawReservation] of Object.entries(raw.reservations)) {
|
|
3924
4278
|
if (!id.startsWith("reservation:") || !validGovernorId(id.slice(12))) return null;
|
|
3925
|
-
const reservation = normalizeGovernorReservation(rawReservation, nowMs);
|
|
4279
|
+
const reservation = normalizeGovernorReservation(rawReservation, nowMs, { legacy: legacyShape });
|
|
3926
4280
|
if (!reservation || id !== `reservation:${reservation.intentId}`) return null;
|
|
3927
4281
|
if (
|
|
3928
4282
|
!prune || reservation.status === "started" || reservation.status === "completed" ||
|
|
@@ -3930,8 +4284,9 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3930
4284
|
) state.reservations[id] = reservation;
|
|
3931
4285
|
}
|
|
3932
4286
|
if (Object.keys(state.reservations).length > GOVERNOR_MAX_RESERVATIONS) return null;
|
|
4287
|
+
if (prune && !legacyShape) retireOwnerGenerations(state);
|
|
3933
4288
|
for (const resource of RATE_RESOURCES) {
|
|
3934
|
-
const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs);
|
|
4289
|
+
const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs, { legacy: legacyShape });
|
|
3935
4290
|
if (claim === undefined) return null;
|
|
3936
4291
|
state.probeClaims[resource] = claim?.leaseUntil > nowMs ? claim : null;
|
|
3937
4292
|
}
|
|
@@ -3940,6 +4295,17 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3940
4295
|
return state;
|
|
3941
4296
|
}
|
|
3942
4297
|
|
|
4298
|
+
function retireOwnerGenerations(state) {
|
|
4299
|
+
const detailedOwners = new Set(Object.values(state.reservations)
|
|
4300
|
+
.filter((reservation) => reservation.receipt !== null)
|
|
4301
|
+
.map((reservation) => reservation.leaseId));
|
|
4302
|
+
for (const leaseId of Object.keys(state.ownerGenerations)) {
|
|
4303
|
+
if (!state.leases[leaseId] && !detailedOwners.has(leaseId)) {
|
|
4304
|
+
delete state.ownerGenerations[leaseId];
|
|
4305
|
+
}
|
|
4306
|
+
}
|
|
4307
|
+
}
|
|
4308
|
+
|
|
3943
4309
|
// Reads a still-running older pane's file as evidence. A version number alone
|
|
3944
4310
|
// is not enough to identify a document: versions 2 and 3 share a *shape* that
|
|
3945
4311
|
// version 4 replaced, so accepting their version against the current key set
|
|
@@ -4048,8 +4414,288 @@ function migrateGovernorState(raw, nowMs) {
|
|
|
4048
4414
|
: adaptPreSplitGovernorShape(raw);
|
|
4049
4415
|
if (!shaped) return null;
|
|
4050
4416
|
if (!isRecord(shaped.fairness)) shaped.fairness = { manualStreak: 0 };
|
|
4051
|
-
shaped.version =
|
|
4052
|
-
|
|
4417
|
+
shaped.version = 6;
|
|
4418
|
+
const legacy = normalizeGovernorState(dropSupersededTabCosts(shaped), nowMs, {
|
|
4419
|
+
prune: false,
|
|
4420
|
+
acceptVersion: 6,
|
|
4421
|
+
});
|
|
4422
|
+
return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
|
|
4423
|
+
}
|
|
4424
|
+
|
|
4425
|
+
function migrateLegacyGovernorReceipts(legacy, nowMs) {
|
|
4426
|
+
const state = { ...legacy, version: GOVERNOR_STATE_VERSION,
|
|
4427
|
+
revision: 0,
|
|
4428
|
+
controlReceipts: { core: null, graphql: null },
|
|
4429
|
+
debt: emptyGovernorState().debt,
|
|
4430
|
+
debtGroups: {},
|
|
4431
|
+
ownerGenerations: {},
|
|
4432
|
+
importMarkers: {},
|
|
4433
|
+
reservations: {},
|
|
4434
|
+
};
|
|
4435
|
+
state.probeClaims = { core: null, graphql: null };
|
|
4436
|
+
for (const resource of RATE_RESOURCES) {
|
|
4437
|
+
if (legacy.probeClaims[resource]) {
|
|
4438
|
+
state.observers[resource] = { ...state.observers[resource], outcome: "idle", nextAt: nowMs };
|
|
4439
|
+
}
|
|
4440
|
+
}
|
|
4441
|
+
const units = { core: 0, graphql: 0 };
|
|
4442
|
+
const counts = { core: 0, graphql: 0 };
|
|
4443
|
+
for (const reservation of Object.values(legacy.reservations)) {
|
|
4444
|
+
for (const resource of RATE_RESOURCES) {
|
|
4445
|
+
const charge = reservationCost(reservation, resource, legacy.leases, nowMs);
|
|
4446
|
+
if (!Number.isSafeInteger(charge) || !Number.isSafeInteger(units[resource] + charge)) return null;
|
|
4447
|
+
units[resource] += charge;
|
|
4448
|
+
if (charge > 0) counts[resource] += 1;
|
|
4449
|
+
}
|
|
4450
|
+
}
|
|
4451
|
+
if (RATE_RESOURCES.some((resource) => units[resource] > 0)) {
|
|
4452
|
+
const nonce = governorId();
|
|
4453
|
+
state.debtGroups[nonce] = {
|
|
4454
|
+
nonce, ownerNonce: null, ownerPid: null, ownerBirth: null, bootId: coordinationBootId(),
|
|
4455
|
+
unknownChild: true, sealed: true, quiescent: false,
|
|
4456
|
+
units, counts, at: nowMs, barrierAt: null,
|
|
4457
|
+
barrierRevisions: { core: 0, graphql: 0 },
|
|
4458
|
+
};
|
|
4459
|
+
for (const resource of RATE_RESOURCES) {
|
|
4460
|
+
state.debt[resource].unresolvedUnits = units[resource];
|
|
4461
|
+
state.debt[resource].unresolvedCount = counts[resource];
|
|
4462
|
+
state.debt[resource].revision = 1;
|
|
4463
|
+
}
|
|
4464
|
+
}
|
|
4465
|
+
return normalizeGovernorState(state, nowMs);
|
|
4466
|
+
}
|
|
4467
|
+
|
|
4468
|
+
const UNKNOWN_DEBT_GROUP_ID = "ffffffff-ffff-4fff-8fff-ffffffffffff";
|
|
4469
|
+
|
|
4470
|
+
let cachedCoordinationBootId;
|
|
4471
|
+
let cachedCoordinationProcessBirth;
|
|
4472
|
+
function coordinationBootId() {
|
|
4473
|
+
if (cachedCoordinationBootId !== undefined) return cachedCoordinationBootId;
|
|
4474
|
+
try {
|
|
4475
|
+
const value = process.platform === "linux"
|
|
4476
|
+
? readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()
|
|
4477
|
+
: process.platform === "darwin"
|
|
4478
|
+
? execFileSync("/usr/sbin/sysctl", ["-n", "kern.boottime"], { encoding: "utf8", timeout: 1000 }).trim()
|
|
4479
|
+
: "";
|
|
4480
|
+
cachedCoordinationBootId = value.length > 0 && value.length <= 128 ? value : null;
|
|
4481
|
+
} catch {
|
|
4482
|
+
cachedCoordinationBootId = null;
|
|
4483
|
+
}
|
|
4484
|
+
return cachedCoordinationBootId;
|
|
4485
|
+
}
|
|
4486
|
+
|
|
4487
|
+
function coordinationProcessBirth(pid = process.pid) {
|
|
4488
|
+
if (pid === process.pid && cachedCoordinationProcessBirth !== undefined) {
|
|
4489
|
+
return cachedCoordinationProcessBirth;
|
|
4490
|
+
}
|
|
4491
|
+
let value = null;
|
|
4492
|
+
try {
|
|
4493
|
+
if (process.platform === "linux") {
|
|
4494
|
+
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
4495
|
+
const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/);
|
|
4496
|
+
value = fields[19] ?? null; // starttime is field 22, after pid and comm.
|
|
4497
|
+
} else if (process.platform === "darwin") {
|
|
4498
|
+
value = execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], {
|
|
4499
|
+
encoding: "utf8", timeout: 1000,
|
|
4500
|
+
}).trim() || null;
|
|
4501
|
+
}
|
|
4502
|
+
} catch { /* an unavailable birth identity is explicitly unknown */ }
|
|
4503
|
+
if (value && value.length > 128) value = null;
|
|
4504
|
+
if (pid === process.pid) cachedCoordinationProcessBirth = value;
|
|
4505
|
+
return value;
|
|
4506
|
+
}
|
|
4507
|
+
|
|
4508
|
+
function debtGroupTarget(state, receipt, groupId = null) {
|
|
4509
|
+
const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
|
|
4510
|
+
const count = Object.keys(state.debtGroups).length;
|
|
4511
|
+
const owned = groupId ? state.debtGroups[groupId] : null;
|
|
4512
|
+
if (owned && !owned.quiescent && owned.bootId === incomingBootId &&
|
|
4513
|
+
(!receipt.receipt || owned.ownerNonce === null ||
|
|
4514
|
+
owned.ownerNonce === receipt.receipt.ownerNonce)) return groupId;
|
|
4515
|
+
if (groupId && !owned && count < GOVERNOR_MAX_DEBT_GROUPS - 1) return groupId;
|
|
4516
|
+
const shared = Object.entries(state.debtGroups).find(([, group]) =>
|
|
4517
|
+
!group.quiescent && group.ownerNonce === null && group.unknownChild &&
|
|
4518
|
+
group.bootId === incomingBootId);
|
|
4519
|
+
if (shared) return shared[0];
|
|
4520
|
+
if (count >= GOVERNOR_MAX_DEBT_GROUPS) return null;
|
|
4521
|
+
return state.debtGroups[UNKNOWN_DEBT_GROUP_ID] ? governorId() : UNKNOWN_DEBT_GROUP_ID;
|
|
4522
|
+
}
|
|
4523
|
+
|
|
4524
|
+
function transferReceiptToDebt(state, receipt, nowMs, groupId = null) {
|
|
4525
|
+
const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
4526
|
+
reservationCost(receipt, resource, state.leases, nowMs)]));
|
|
4527
|
+
if (RATE_RESOURCES.every((resource) => units[resource] === 0)) return true;
|
|
4528
|
+
const id = debtGroupTarget(state, receipt, groupId);
|
|
4529
|
+
if (id === null) return false;
|
|
4530
|
+
const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
|
|
4531
|
+
const dedicated = id === groupId;
|
|
4532
|
+
const group = state.debtGroups[id] ?? {
|
|
4533
|
+
nonce: governorId(), ownerNonce: null, ownerPid: null, ownerBirth: null,
|
|
4534
|
+
bootId: incomingBootId,
|
|
4535
|
+
unknownChild: !dedicated || !receipt.receipt, sealed: true, quiescent: false,
|
|
4536
|
+
units: { core: 0, graphql: 0 }, counts: { core: 0, graphql: 0 },
|
|
4537
|
+
at: nowMs, barrierAt: null,
|
|
4538
|
+
barrierRevisions: { core: 0, graphql: 0 },
|
|
4539
|
+
};
|
|
4540
|
+
if (group.quiescent) return false;
|
|
4541
|
+
if (receipt.receipt && dedicated) {
|
|
4542
|
+
if (group.ownerNonce !== null && group.ownerNonce !== receipt.receipt.ownerNonce) return false;
|
|
4543
|
+
group.ownerNonce = receipt.receipt.ownerNonce;
|
|
4544
|
+
group.ownerPid = receipt.receipt.ownerPid;
|
|
4545
|
+
group.ownerBirth = receipt.receipt.ownerBirth;
|
|
4546
|
+
group.bootId = receipt.receipt.bootId;
|
|
4547
|
+
group.unknownChild ||= receipt.receipt.dispatches.some((dispatch) =>
|
|
4548
|
+
dispatch.terminalAt === null);
|
|
4549
|
+
}
|
|
4550
|
+
for (const resource of RATE_RESOURCES) {
|
|
4551
|
+
const amount = units[resource];
|
|
4552
|
+
if (!Number.isSafeInteger(amount) || amount < 0 ||
|
|
4553
|
+
!Number.isSafeInteger(state.debt[resource].unresolvedUnits + amount) ||
|
|
4554
|
+
!Number.isSafeInteger(group.units[resource] + amount) ||
|
|
4555
|
+
!Number.isSafeInteger(state.debt[resource].unresolvedCount + (amount > 0 ? 1 : 0)) ||
|
|
4556
|
+
!Number.isSafeInteger(state.debt[resource].revision + 1) ||
|
|
4557
|
+
!Number.isSafeInteger(group.counts[resource] + (amount > 0 ? 1 : 0))) return false;
|
|
4558
|
+
}
|
|
4559
|
+
for (const resource of RATE_RESOURCES) {
|
|
4560
|
+
const amount = units[resource];
|
|
4561
|
+
state.debt[resource].unresolvedUnits += amount;
|
|
4562
|
+
state.debt[resource].unresolvedCount += amount > 0 ? 1 : 0;
|
|
4563
|
+
state.debt[resource].revision += 1;
|
|
4564
|
+
group.units[resource] += amount;
|
|
4565
|
+
group.counts[resource] += amount > 0 ? 1 : 0;
|
|
4566
|
+
}
|
|
4567
|
+
state.debtGroups[id] = group;
|
|
4568
|
+
return true;
|
|
4569
|
+
}
|
|
4570
|
+
|
|
4571
|
+
function makeDebtGroupQuiescent(state, group, at) {
|
|
4572
|
+
for (const resource of RATE_RESOURCES) {
|
|
4573
|
+
const debt = state.debt[resource];
|
|
4574
|
+
if (debt.unresolvedUnits < group.units[resource] || debt.unresolvedCount < group.counts[resource] ||
|
|
4575
|
+
!Number.isSafeInteger(debt.quiescentUnits + group.units[resource]) ||
|
|
4576
|
+
!Number.isSafeInteger(debt.quiescentCount + group.counts[resource]) ||
|
|
4577
|
+
!Number.isSafeInteger(debt.revision + 1)) return false;
|
|
4578
|
+
}
|
|
4579
|
+
for (const resource of RATE_RESOURCES) {
|
|
4580
|
+
const debt = state.debt[resource];
|
|
4581
|
+
debt.unresolvedUnits -= group.units[resource];
|
|
4582
|
+
debt.unresolvedCount -= group.counts[resource];
|
|
4583
|
+
debt.quiescentUnits += group.units[resource];
|
|
4584
|
+
debt.quiescentCount += group.counts[resource];
|
|
4585
|
+
debt.revision += 1;
|
|
4586
|
+
}
|
|
4587
|
+
group.quiescent = true;
|
|
4588
|
+
group.barrierAt = at;
|
|
4589
|
+
group.barrierRevisions = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
4590
|
+
resource, state.debt[resource].revision,
|
|
4591
|
+
]));
|
|
4592
|
+
return true;
|
|
4593
|
+
}
|
|
4594
|
+
|
|
4595
|
+
function retireControlReceiptToDebt(state, resource, at) {
|
|
4596
|
+
const control = state.controlReceipts[resource];
|
|
4597
|
+
if (!control) return true;
|
|
4598
|
+
// A control receipt without an issued dispatch cannot have started HTTP.
|
|
4599
|
+
// Removing it also fences a late owner before it can issue a child.
|
|
4600
|
+
if (control.receipt.dispatches.length === 0) {
|
|
4601
|
+
state.controlReceipts[resource] = null;
|
|
4602
|
+
return true;
|
|
4603
|
+
}
|
|
4604
|
+
const generation = control.receipt.generation;
|
|
4605
|
+
if (!transferReceiptToDebt(state, control, at, generation)) return false;
|
|
4606
|
+
const group = state.debtGroups[generation];
|
|
4607
|
+
if (group && !group.unknownChild && group.ownerPid !== null &&
|
|
4608
|
+
group.ownerBirth !== null && group.bootId !== null &&
|
|
4609
|
+
!makeDebtGroupQuiescent(state, group, at)) return false;
|
|
4610
|
+
state.controlReceipts[resource] = null;
|
|
4611
|
+
return true;
|
|
4612
|
+
}
|
|
4613
|
+
|
|
4614
|
+
function reconcileBootDebt(state, at) {
|
|
4615
|
+
const bootId = coordinationBootId();
|
|
4616
|
+
if (bootId === null) return { ok: true, changed: false };
|
|
4617
|
+
let changed = false;
|
|
4618
|
+
for (const group of Object.values(state.debtGroups)) {
|
|
4619
|
+
if (group.quiescent || group.bootId === null || group.bootId === bootId) continue;
|
|
4620
|
+
if (!makeDebtGroupQuiescent(state, group, at)) {
|
|
4621
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4622
|
+
}
|
|
4623
|
+
changed = true;
|
|
4624
|
+
}
|
|
4625
|
+
return { ok: true, changed };
|
|
4626
|
+
}
|
|
4627
|
+
|
|
4628
|
+
function compactGovernorReceipts(state, nowMs) {
|
|
4629
|
+
const sealed = new Set();
|
|
4630
|
+
const full = Object.keys(state.reservations).length >= GOVERNOR_MAX_RESERVATIONS;
|
|
4631
|
+
let changed = false;
|
|
4632
|
+
const dueGenerations = new Set(Object.values(state.reservations)
|
|
4633
|
+
.filter((reservation) => reservation.receipt && (
|
|
4634
|
+
reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
|
|
4635
|
+
full && reservation.status === "completed"))
|
|
4636
|
+
.map((reservation) => reservation.receipt.generation));
|
|
4637
|
+
// A full debt journal must not prevent a claimed observer from retiring
|
|
4638
|
+
// older quiescent groups. Keep the detailed receipts until space is proven.
|
|
4639
|
+
const candidates = Object.values(state.reservations).filter((reservation) => {
|
|
4640
|
+
const groupId = reservation.receipt?.generation ?? null;
|
|
4641
|
+
return (full && reservation.status === "completed" && !reservation.receipt ||
|
|
4642
|
+
groupId && dueGenerations.has(groupId) &&
|
|
4643
|
+
!(reservation.status === "started" && reservation.receipt.dispatches.length === 0)) &&
|
|
4644
|
+
RATE_RESOURCES.some((resource) =>
|
|
4645
|
+
reservationCost(reservation, resource, state.leases, nowMs) > 0);
|
|
4646
|
+
});
|
|
4647
|
+
if (candidates.length > 0) {
|
|
4648
|
+
const trial = structuredClone(state);
|
|
4649
|
+
for (const reservation of candidates) {
|
|
4650
|
+
const groupId = reservation.receipt?.generation ?? null;
|
|
4651
|
+
if (debtGroupTarget(trial, reservation, groupId) === null) return { ok: true, changed: false };
|
|
4652
|
+
if (!transferReceiptToDebt(trial, reservation, nowMs, groupId)) {
|
|
4653
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4654
|
+
}
|
|
4655
|
+
}
|
|
4656
|
+
}
|
|
4657
|
+
if (full) {
|
|
4658
|
+
for (const [id, reservation] of Object.entries(state.reservations)) {
|
|
4659
|
+
if (reservation.status !== "completed" || reservation.receipt) continue;
|
|
4660
|
+
if (!transferReceiptToDebt(state, reservation, nowMs)) {
|
|
4661
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4662
|
+
}
|
|
4663
|
+
delete state.reservations[id];
|
|
4664
|
+
changed = true;
|
|
4665
|
+
}
|
|
4666
|
+
}
|
|
4667
|
+
for (const reservation of Object.values(state.reservations)) {
|
|
4668
|
+
if (reservation.receipt && (
|
|
4669
|
+
reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
|
|
4670
|
+
full && reservation.status === "completed")) {
|
|
4671
|
+
sealed.add(reservation.receipt.generation);
|
|
4672
|
+
}
|
|
4673
|
+
}
|
|
4674
|
+
if (sealed.size === 0) {
|
|
4675
|
+
if (changed) retireOwnerGenerations(state);
|
|
4676
|
+
return { ok: true, changed };
|
|
4677
|
+
}
|
|
4678
|
+
// Seal the complete owner generation. A late completion of one receipt cannot
|
|
4679
|
+
// make another already-authorized child in that generation disappear.
|
|
4680
|
+
for (const generation of sealed) {
|
|
4681
|
+
for (const [id, reservation] of Object.entries(state.reservations)) {
|
|
4682
|
+
if (reservation.receipt?.generation !== generation) continue;
|
|
4683
|
+
if (!(reservation.status === "started" && reservation.receipt.dispatches.length === 0) &&
|
|
4684
|
+
!transferReceiptToDebt(state, reservation, nowMs, generation)) {
|
|
4685
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4686
|
+
}
|
|
4687
|
+
delete state.reservations[id];
|
|
4688
|
+
state.ownerGenerations[reservation.leaseId] = governorId();
|
|
4689
|
+
}
|
|
4690
|
+
const group = state.debtGroups[generation];
|
|
4691
|
+
if (group && !group.unknownChild && group.ownerPid !== null &&
|
|
4692
|
+
group.ownerBirth !== null && group.bootId !== null &&
|
|
4693
|
+
!makeDebtGroupQuiescent(state, group, nowMs)) {
|
|
4694
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4695
|
+
}
|
|
4696
|
+
}
|
|
4697
|
+
retireOwnerGenerations(state);
|
|
4698
|
+
return { ok: true, changed: true };
|
|
4053
4699
|
}
|
|
4054
4700
|
|
|
4055
4701
|
function migrateV1GovernorState(raw, nowMs) {
|
|
@@ -4058,7 +4704,7 @@ function migrateV1GovernorState(raw, nowMs) {
|
|
|
4058
4704
|
"intents", "reservations", "manualProbe",
|
|
4059
4705
|
]) || raw.version !== 1 || !isRecord(raw.budgets) || !isRecord(raw.reservations)) return null;
|
|
4060
4706
|
const migrated = structuredClone(raw);
|
|
4061
|
-
migrated.version =
|
|
4707
|
+
migrated.version = 6;
|
|
4062
4708
|
migrated.observers = { core: { etag: null, outcome: "idle", at: 0, nextAt: 0 } };
|
|
4063
4709
|
// The v1 core value came from /rate_limit, which is not authoritative. Do not
|
|
4064
4710
|
// preserve it as bootstrap evidence. GraphQL remains valid.
|
|
@@ -4086,7 +4732,8 @@ function migrateV1GovernorState(raw, nowMs) {
|
|
|
4086
4732
|
]));
|
|
4087
4733
|
migrated.probeClaims = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, null]));
|
|
4088
4734
|
migrated.fairness = { manualStreak: 0 };
|
|
4089
|
-
|
|
4735
|
+
const legacy = normalizeGovernorState(migrated, nowMs, { prune: false, acceptVersion: 6 });
|
|
4736
|
+
return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
|
|
4090
4737
|
}
|
|
4091
4738
|
|
|
4092
4739
|
function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
|
|
@@ -4099,11 +4746,21 @@ function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
|
|
|
4099
4746
|
: { ok: false, reason: "unwritable" };
|
|
4100
4747
|
}
|
|
4101
4748
|
try {
|
|
4749
|
+
if (Buffer.byteLength(raw) > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "capacity" };
|
|
4102
4750
|
const parsed = JSON.parse(raw);
|
|
4103
4751
|
const normalized = normalizeGovernorState(parsed, nowMs);
|
|
4104
4752
|
if (normalized) return { ok: true, value: normalized };
|
|
4105
4753
|
const migrated = migrateGovernorState(parsed, nowMs);
|
|
4106
4754
|
if (!migrated) return { ok: false, reason: "corrupt" };
|
|
4755
|
+
if (persistMigration && parsed.version === 6) {
|
|
4756
|
+
const backupPath = `${path}.pre-v7.backup`;
|
|
4757
|
+
try {
|
|
4758
|
+
writeFileSync(backupPath, raw, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
4759
|
+
chmodSync(backupPath, 0o600);
|
|
4760
|
+
} catch (error) {
|
|
4761
|
+
if (error?.code !== "EEXIST") return { ok: false, reason: "unwritable" };
|
|
4762
|
+
}
|
|
4763
|
+
}
|
|
4107
4764
|
if (!persistMigration) return { ok: true, value: migrated, migrated: true };
|
|
4108
4765
|
// Read-only callers persist the exact migration while they still hold the
|
|
4109
4766
|
// scope lock. Mutating callers use this same value for their final write.
|
|
@@ -4118,13 +4775,25 @@ function writeGovernorState(path, state) {
|
|
|
4118
4775
|
const parent = dirname(path);
|
|
4119
4776
|
let tempPath = null;
|
|
4120
4777
|
try {
|
|
4778
|
+
if (state?.version === GOVERNOR_STATE_VERSION) {
|
|
4779
|
+
if (!Number.isSafeInteger(state.revision) || state.revision === Number.MAX_SAFE_INTEGER) {
|
|
4780
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4781
|
+
}
|
|
4782
|
+
state.revision += 1;
|
|
4783
|
+
}
|
|
4784
|
+
const serialized = serializeGovernorState(state);
|
|
4785
|
+
if (state?.version === GOVERNOR_STATE_VERSION &&
|
|
4786
|
+
Buffer.byteLength(serialized) > GOVERNOR_MAX_LEDGER_BYTES) {
|
|
4787
|
+
return { ok: false, reason: "capacity" };
|
|
4788
|
+
}
|
|
4121
4789
|
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
4122
4790
|
chmodSync(parent, 0o700);
|
|
4123
4791
|
tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
4124
|
-
writeFileSync(tempPath,
|
|
4792
|
+
writeFileSync(tempPath, serialized, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
4125
4793
|
chmodSync(tempPath, 0o600);
|
|
4794
|
+
// The private temporary inode already has mode 0600. Rename is the commit
|
|
4795
|
+
// point: no fallible operation may report failure after state replacement.
|
|
4126
4796
|
renameSync(tempPath, path);
|
|
4127
|
-
chmodSync(path, 0o600);
|
|
4128
4797
|
return { ok: true };
|
|
4129
4798
|
} catch {
|
|
4130
4799
|
if (tempPath !== null) {
|
|
@@ -4134,6 +4803,15 @@ function writeGovernorState(path, state) {
|
|
|
4134
4803
|
}
|
|
4135
4804
|
}
|
|
4136
4805
|
|
|
4806
|
+
function writeGovernorQuotaState(path, state, nowMs) {
|
|
4807
|
+
const normalized = normalizeGovernorState(state, nowMs, { prune: false });
|
|
4808
|
+
if (!normalized) return { ok: false, reason: "corrupt" };
|
|
4809
|
+
if (!governorAccountingSafe(normalized, nowMs)) {
|
|
4810
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4811
|
+
}
|
|
4812
|
+
return writeGovernorState(path, normalized);
|
|
4813
|
+
}
|
|
4814
|
+
|
|
4137
4815
|
function lockOwner(path) {
|
|
4138
4816
|
try {
|
|
4139
4817
|
const owner = JSON.parse(readFileSync(path, "utf8"));
|
|
@@ -4443,9 +5121,18 @@ function mutateGovernor(scope, nowMs, mutate) {
|
|
|
4443
5121
|
const loaded = readGovernorState(scope.path, at, { persistMigration: false });
|
|
4444
5122
|
if (!loaded.ok) return loaded;
|
|
4445
5123
|
const state = loaded.value;
|
|
5124
|
+
if (!governorAccountingSafe(state, at)) return { ok: false, reason: "accounting-overflow" };
|
|
5125
|
+
const oldBootReconciled = reconcileBootDebt(state, at);
|
|
5126
|
+
if (!oldBootReconciled.ok) return oldBootReconciled;
|
|
5127
|
+
const compacted = compactGovernorReceipts(state, at);
|
|
5128
|
+
if (!compacted.ok) return compacted;
|
|
5129
|
+
const bootReconciled = reconcileBootDebt(state, at);
|
|
5130
|
+
if (!bootReconciled.ok) return bootReconciled;
|
|
4446
5131
|
// A mutator can reject without writing. Keep an untouched migration so
|
|
4447
5132
|
// that even that path upgrades the file once before releasing the lock.
|
|
4448
|
-
const migration = loaded.migrated
|
|
5133
|
+
const migration = loaded.migrated || compacted.changed ||
|
|
5134
|
+
oldBootReconciled.changed || bootReconciled.changed
|
|
5135
|
+
? structuredClone(state) : null;
|
|
4449
5136
|
const returnAfterMigration = (result) => {
|
|
4450
5137
|
if (migration === null) return result;
|
|
4451
5138
|
const written = writeGovernorState(scope.path, migration);
|
|
@@ -4453,6 +5140,7 @@ function mutateGovernor(scope, nowMs, mutate) {
|
|
|
4453
5140
|
};
|
|
4454
5141
|
const value = mutate(state, at);
|
|
4455
5142
|
if (value?.ok === false && value.write !== true) return returnAfterMigration(value);
|
|
5143
|
+
if (!governorAccountingSafe(state, at)) return returnAfterMigration({ ok: false, reason: "accounting-overflow" });
|
|
4456
5144
|
const normalized = normalizeGovernorState(state, at);
|
|
4457
5145
|
if (!normalized) return returnAfterMigration({ ok: false, reason: "corrupt" });
|
|
4458
5146
|
const written = writeGovernorState(scope.path, normalized);
|
|
@@ -4481,7 +5169,7 @@ const GOVERNOR_MAX_ATOMIC_COST = Math.max(...Object.values(OPERATION_COSTS)
|
|
|
4481
5169
|
function returnPacingCredit(state, resource, unusedCost, nowMs) {
|
|
4482
5170
|
const budget = state.budgets[resource];
|
|
4483
5171
|
if (!budget || !(unusedCost > 0)) return;
|
|
4484
|
-
const decision =
|
|
5172
|
+
const decision = governorResourceDecision(state, resource, nowMs, 0);
|
|
4485
5173
|
const callsPerMs = decision?.callsPerMs;
|
|
4486
5174
|
if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) return;
|
|
4487
5175
|
const credit = Math.min(unusedCost, GOVERNOR_MAX_ATOMIC_COST) / callsPerMs;
|
|
@@ -4544,6 +5232,10 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
4544
5232
|
leases: state.leases,
|
|
4545
5233
|
budgets: state.budgets,
|
|
4546
5234
|
reservations: Object.entries(state.reservations).map(([id, reservation]) => ({ id, ...reservation })),
|
|
5235
|
+
debtCosts: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
5236
|
+
state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
|
|
5237
|
+
reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs),
|
|
5238
|
+
])),
|
|
4547
5239
|
lanes,
|
|
4548
5240
|
cursors,
|
|
4549
5241
|
nowMs,
|
|
@@ -4572,6 +5264,7 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
4572
5264
|
startedAt: null,
|
|
4573
5265
|
completedAt: null,
|
|
4574
5266
|
outcome: null,
|
|
5267
|
+
receipt: null,
|
|
4575
5268
|
};
|
|
4576
5269
|
reservationCount += 1;
|
|
4577
5270
|
delete state.intents[grant.intentId];
|
|
@@ -4676,6 +5369,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4676
5369
|
dueAtCoreReset,
|
|
4677
5370
|
);
|
|
4678
5371
|
if (nextAt > at) return { value: { status: "waiting", nextAt } };
|
|
5372
|
+
if (resource === "graphql" && state.controlReceipts.graphql) {
|
|
5373
|
+
// The old claim's lease has expired. Keep an issued child charged as
|
|
5374
|
+
// debt, and fence the old capability so a new observer can make progress.
|
|
5375
|
+
if (!retireControlReceiptToDebt(state, "graphql", at)) {
|
|
5376
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5377
|
+
}
|
|
5378
|
+
}
|
|
4679
5379
|
const nonce = randomUUID();
|
|
4680
5380
|
const startedReservationIds = Object.entries(state.reservations)
|
|
4681
5381
|
// An expired owner can no longer settle its request. Keep that uncertain
|
|
@@ -4686,6 +5386,36 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4686
5386
|
state.leases[reservation.leaseId]?.expiresAt > at &&
|
|
4687
5387
|
reservationCost(reservation, resource, state.leases, at) > 0)
|
|
4688
5388
|
.map(([id]) => id);
|
|
5389
|
+
const quiescentGroups = Object.values(state.debtGroups).filter((group) =>
|
|
5390
|
+
group.quiescent && group.barrierAt < at &&
|
|
5391
|
+
(group.units[resource] > 0 || group.counts[resource] > 0));
|
|
5392
|
+
const barrierAt = Math.max(0, ...quiescentGroups.map((group) => group.barrierAt));
|
|
5393
|
+
const debtUnits = quiescentGroups.reduce((sum, group) => sum + group.units[resource], 0);
|
|
5394
|
+
const debtCount = quiescentGroups.reduce((sum, group) => sum + group.counts[resource], 0);
|
|
5395
|
+
if (!Number.isSafeInteger(debtUnits) || !Number.isSafeInteger(debtCount)) {
|
|
5396
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5397
|
+
}
|
|
5398
|
+
const debtSnapshot = {
|
|
5399
|
+
units: debtUnits,
|
|
5400
|
+
count: debtCount,
|
|
5401
|
+
barrierAt,
|
|
5402
|
+
revision: state.debt[resource].revision,
|
|
5403
|
+
};
|
|
5404
|
+
if (resource === "graphql") {
|
|
5405
|
+
state.controlReceipts.graphql = {
|
|
5406
|
+
leaseId, intentId: nonce, costs: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS },
|
|
5407
|
+
actualCosts: null, accountedCosts: { core: 0, graphql: 0 },
|
|
5408
|
+
notBefore: at, status: "started",
|
|
5409
|
+
epochs: { core: null, graphql: state.epochs.graphql },
|
|
5410
|
+
startedAt: at, completedAt: null, outcome: null,
|
|
5411
|
+
receipt: {
|
|
5412
|
+
scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
|
|
5413
|
+
ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
|
|
5414
|
+
deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
|
|
5415
|
+
allowance: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS }, dispatches: [],
|
|
5416
|
+
},
|
|
5417
|
+
};
|
|
5418
|
+
}
|
|
4689
5419
|
state.probeClaims[resource] = {
|
|
4690
5420
|
ownerLeaseId: leaseId,
|
|
4691
5421
|
nonce,
|
|
@@ -4693,6 +5423,7 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4693
5423
|
nextAt: at,
|
|
4694
5424
|
claimAt: at,
|
|
4695
5425
|
startedReservationIds,
|
|
5426
|
+
debtSnapshot,
|
|
4696
5427
|
};
|
|
4697
5428
|
state.observers[resource].outcome = "waiting";
|
|
4698
5429
|
state.observers[resource].at = at;
|
|
@@ -4704,6 +5435,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4704
5435
|
leaseUntil: state.probeClaims[resource].leaseUntil,
|
|
4705
5436
|
startedReservationIds,
|
|
4706
5437
|
coreEtag: state.observers[resource].etag,
|
|
5438
|
+
...(resource === "graphql" ? { receiptCapability: {
|
|
5439
|
+
reservationId: "control:graphql", controlResource: "graphql", leaseId,
|
|
5440
|
+
scopeHash: scope.hash,
|
|
5441
|
+
ownerNonce: state.controlReceipts.graphql.receipt.ownerNonce,
|
|
5442
|
+
generation: state.controlReceipts.graphql.receipt.generation,
|
|
5443
|
+
deadline: state.controlReceipts.graphql.receipt.deadline,
|
|
5444
|
+
} } : {}),
|
|
4707
5445
|
} };
|
|
4708
5446
|
});
|
|
4709
5447
|
}
|
|
@@ -4785,6 +5523,34 @@ function budgetFromObservation(raw, previous, nowMs, {
|
|
|
4785
5523
|
return { status: "accepted", budget: next, epochChanged };
|
|
4786
5524
|
}
|
|
4787
5525
|
|
|
5526
|
+
function retireClaimedQuiescentDebt(state, resource, claim) {
|
|
5527
|
+
const snapshot = claim.debtSnapshot;
|
|
5528
|
+
if (snapshot.units === 0 && snapshot.count === 0) return true;
|
|
5529
|
+
if (claim.claimAt <= snapshot.barrierAt) return true;
|
|
5530
|
+
const eligible = Object.entries(state.debtGroups).filter(([, group]) =>
|
|
5531
|
+
group.quiescent && group.barrierAt < claim.claimAt &&
|
|
5532
|
+
group.barrierRevisions[resource] <= snapshot.revision &&
|
|
5533
|
+
(group.units[resource] > 0 || group.counts[resource] > 0));
|
|
5534
|
+
const units = eligible.reduce((total, [, group]) => total + group.units[resource], 0);
|
|
5535
|
+
const count = eligible.reduce((total, [, group]) => total + group.counts[resource], 0);
|
|
5536
|
+
const debt = state.debt[resource];
|
|
5537
|
+
if (!Number.isSafeInteger(units) || !Number.isSafeInteger(count) ||
|
|
5538
|
+
units !== snapshot.units || count !== snapshot.count ||
|
|
5539
|
+
debt.quiescentUnits < units || debt.quiescentCount < count ||
|
|
5540
|
+
!Number.isSafeInteger(debt.revision + 1)) return false;
|
|
5541
|
+
debt.quiescentUnits -= units;
|
|
5542
|
+
debt.quiescentCount -= count;
|
|
5543
|
+
debt.revision += 1;
|
|
5544
|
+
for (const [id, group] of eligible) {
|
|
5545
|
+
group.units[resource] = 0;
|
|
5546
|
+
group.counts[resource] = 0;
|
|
5547
|
+
if (RATE_RESOURCES.every((name) => group.units[name] === 0 && group.counts[name] === 0)) {
|
|
5548
|
+
delete state.debtGroups[id];
|
|
5549
|
+
}
|
|
5550
|
+
}
|
|
5551
|
+
return true;
|
|
5552
|
+
}
|
|
5553
|
+
|
|
4788
5554
|
function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
|
|
4789
5555
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
4790
5556
|
const claim = state.probeClaims[resource];
|
|
@@ -4833,6 +5599,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
|
|
|
4833
5599
|
}
|
|
4834
5600
|
}
|
|
4835
5601
|
if (changedResources.length === 0) return { ok: false, reason: "corrupt" };
|
|
5602
|
+
if (!retireClaimedQuiescentDebt(state, resource, claim)) {
|
|
5603
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5604
|
+
}
|
|
4836
5605
|
const completedBeforeClaim = Object.entries(state.reservations)
|
|
4837
5606
|
.filter(([, reservation]) =>
|
|
4838
5607
|
reservation.status === "completed" && reservation.completedAt < claim.claimAt);
|
|
@@ -4884,6 +5653,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
|
|
|
4884
5653
|
state.budgets = nextBudgets;
|
|
4885
5654
|
state.epochs = nextEpochs;
|
|
4886
5655
|
state.probeClaims[resource] = null;
|
|
5656
|
+
if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
|
|
5657
|
+
state.controlReceipts.graphql = null;
|
|
5658
|
+
}
|
|
4887
5659
|
{
|
|
4888
5660
|
const published = nextBudgets[resource];
|
|
4889
5661
|
state.observers[resource] = {
|
|
@@ -4944,6 +5716,15 @@ function failProbeClaim(scope, leaseId, nonce, nowMs, resource) {
|
|
|
4944
5716
|
// also marked core failed would erase authority core had legitimately
|
|
4945
5717
|
// established, and stall a lane with nothing wrong with it.
|
|
4946
5718
|
state.probeClaims[resource] = null;
|
|
5719
|
+
if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
|
|
5720
|
+
const control = state.controlReceipts.graphql;
|
|
5721
|
+
if (control.receipt.dispatches.length === 0) state.controlReceipts.graphql = null;
|
|
5722
|
+
else if (control.receipt.dispatches.every((dispatch) => dispatch.terminalAt !== null)) {
|
|
5723
|
+
if (!retireControlReceiptToDebt(state, "graphql", at)) {
|
|
5724
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5725
|
+
}
|
|
5726
|
+
}
|
|
5727
|
+
}
|
|
4947
5728
|
state.observers[resource] = {
|
|
4948
5729
|
...state.observers[resource],
|
|
4949
5730
|
outcome: "failed",
|
|
@@ -5118,7 +5899,8 @@ function startReservation(scope, reservationId, nowMs) {
|
|
|
5118
5899
|
}
|
|
5119
5900
|
const charged = Object.values(state.reservations).reduce(
|
|
5120
5901
|
(total, item) => total + reservationCost(item, resource, state.leases, at),
|
|
5121
|
-
|
|
5902
|
+
state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
|
|
5903
|
+
reservationCost(state.controlReceipts[resource], resource, state.leases, at),
|
|
5122
5904
|
);
|
|
5123
5905
|
if (budget.remaining - resourceReserve(budget.limit) - charged < 0) {
|
|
5124
5906
|
reservation.status = "cancelled";
|
|
@@ -5127,43 +5909,204 @@ function startReservation(scope, reservationId, nowMs) {
|
|
|
5127
5909
|
}
|
|
5128
5910
|
reservation.status = "started";
|
|
5129
5911
|
reservation.startedAt = at;
|
|
5130
|
-
|
|
5912
|
+
const generation = state.ownerGenerations[reservation.leaseId] ?? governorId();
|
|
5913
|
+
state.ownerGenerations[reservation.leaseId] = generation;
|
|
5914
|
+
reservation.receipt = {
|
|
5915
|
+
scopeHash: scope.hash,
|
|
5916
|
+
ownerNonce: generation,
|
|
5917
|
+
generation,
|
|
5918
|
+
ownerPid: process.pid,
|
|
5919
|
+
ownerBirth: coordinationProcessBirth(),
|
|
5920
|
+
bootId: coordinationBootId(),
|
|
5921
|
+
deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
|
|
5922
|
+
allowance: { ...reservation.costs },
|
|
5923
|
+
dispatches: [],
|
|
5924
|
+
};
|
|
5925
|
+
return { value: { status: "started", reservationId,
|
|
5926
|
+
receiptCapability: receiptCapabilityFromReservation(reservationId, reservation) } };
|
|
5131
5927
|
});
|
|
5132
5928
|
}
|
|
5133
5929
|
|
|
5134
|
-
function
|
|
5930
|
+
function mutateReceiptDispatch(scope, capability, nowMs, mutate) {
|
|
5931
|
+
if (!capability || capability.scopeHash !== scope?.hash ||
|
|
5932
|
+
!validGovernorId(capability.ownerNonce) || !validGovernorId(capability.generation) ||
|
|
5933
|
+
!validGovernorId(capability.leaseId) ||
|
|
5934
|
+
typeof capability.reservationId !== "string") return { ok: false, reason: "receipt-invalid" };
|
|
5135
5935
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
5136
|
-
const
|
|
5137
|
-
if (
|
|
5138
|
-
return { ok: false, reason: "
|
|
5139
|
-
}
|
|
5140
|
-
const
|
|
5141
|
-
?
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5936
|
+
const controlResource = capability.controlResource;
|
|
5937
|
+
if (controlResource !== undefined && !RATE_RESOURCES.includes(controlResource)) {
|
|
5938
|
+
return { ok: false, reason: "receipt-invalid" };
|
|
5939
|
+
}
|
|
5940
|
+
const reservation = controlResource
|
|
5941
|
+
? state.controlReceipts[controlResource] : state.reservations[capability.reservationId];
|
|
5942
|
+
const receipt = reservation?.receipt;
|
|
5943
|
+
if (!receipt || receipt.scopeHash !== capability.scopeHash ||
|
|
5944
|
+
receipt.ownerNonce !== capability.ownerNonce || receipt.generation !== capability.generation ||
|
|
5945
|
+
reservation.leaseId !== capability.leaseId || receipt.deadline !== capability.deadline) {
|
|
5946
|
+
return { ok: false, reason: "compacted" };
|
|
5947
|
+
}
|
|
5948
|
+
return mutate(state, reservation, receipt, at);
|
|
5949
|
+
});
|
|
5950
|
+
}
|
|
5951
|
+
|
|
5952
|
+
function issueGovernorDispatch(scope, capability, operation, nowMs) {
|
|
5953
|
+
const costs = operationCost(operation);
|
|
5954
|
+
if (!costs) return { ok: false, reason: "undeclared" };
|
|
5955
|
+
if ([...terminalizationBacklog.keys()].filter((key) =>
|
|
5956
|
+
key.startsWith(`${scope?.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
|
|
5957
|
+
return { ok: false, reason: "completion-capacity" };
|
|
5958
|
+
}
|
|
5959
|
+
return mutateReceiptDispatch(scope, capability, nowMs, (state, reservation, receipt, at) => {
|
|
5960
|
+
if (reservation.status !== "started" || receipt.deadline <= at ||
|
|
5961
|
+
(!capability.controlResource && (
|
|
5962
|
+
state.ownerGenerations[reservation.leaseId] !== receipt.generation ||
|
|
5963
|
+
state.leases[reservation.leaseId]?.expiresAt <= at ||
|
|
5964
|
+
!state.leases[reservation.leaseId]))) return { ok: false, reason: "receipt-expired" };
|
|
5965
|
+
const issued = { core: 0, graphql: 0 };
|
|
5966
|
+
for (const dispatch of receipt.dispatches) {
|
|
5967
|
+
for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
|
|
5968
|
+
}
|
|
5969
|
+
if (RATE_RESOURCES.some((resource) =>
|
|
5970
|
+
!Number.isSafeInteger(issued[resource] + costs[resource]) ||
|
|
5971
|
+
issued[resource] + costs[resource] > receipt.allowance[resource])) {
|
|
5972
|
+
return { ok: false, reason: "receipt-allowance" };
|
|
5973
|
+
}
|
|
5974
|
+
const sequence = receipt.dispatches.length + 1;
|
|
5975
|
+
if (sequence > 32) return { ok: false, reason: "receipt-allowance" };
|
|
5976
|
+
receipt.dispatches.push({ sequence, operation, costs: { ...costs },
|
|
5977
|
+
issuedAt: at, terminalAt: null, childPid: null, childBirth: null, neverStarted: false });
|
|
5978
|
+
return { value: { sequence, deadline: receipt.deadline } };
|
|
5979
|
+
});
|
|
5980
|
+
}
|
|
5981
|
+
|
|
5982
|
+
function markGovernorDispatchChild(scope, capability, sequence, pid, nowMs) {
|
|
5983
|
+
if (!Number.isSafeInteger(pid) || pid < 1) return { ok: false, reason: "child-unknown" };
|
|
5984
|
+
return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
|
|
5985
|
+
(_state, _reservation, receipt) => {
|
|
5986
|
+
const dispatch = receipt.dispatches[sequence - 1];
|
|
5987
|
+
if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
|
|
5988
|
+
dispatch.childPid = pid;
|
|
5989
|
+
dispatch.childBirth = coordinationProcessBirth(pid);
|
|
5990
|
+
return { value: { status: "recorded" } };
|
|
5991
|
+
});
|
|
5992
|
+
}
|
|
5993
|
+
|
|
5994
|
+
function terminalGovernorDispatch(scope, capability, sequence, nowMs, { neverStarted = false } = {}) {
|
|
5995
|
+
return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
|
|
5996
|
+
(_state, _reservation, receipt, at) => {
|
|
5997
|
+
const dispatch = receipt.dispatches[sequence - 1];
|
|
5998
|
+
if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
|
|
5999
|
+
if (neverStarted && dispatch.childPid !== null) return { ok: false, reason: "child-started" };
|
|
6000
|
+
if (dispatch.terminalAt === null) {
|
|
6001
|
+
dispatch.terminalAt = at;
|
|
6002
|
+
dispatch.neverStarted = neverStarted;
|
|
5161
6003
|
}
|
|
5162
|
-
|
|
5163
|
-
|
|
6004
|
+
return { value: { status: "terminal" } };
|
|
6005
|
+
});
|
|
6006
|
+
}
|
|
6007
|
+
|
|
6008
|
+
const activeOwnerDispatches = new Map();
|
|
6009
|
+
function ownerDispatchKey(scope, capability) {
|
|
6010
|
+
return `${scope.hash}:${capability.generation}`;
|
|
6011
|
+
}
|
|
6012
|
+
|
|
6013
|
+
function acknowledgeSealedGeneration(scope, capability, sealedNonce, nowMs) {
|
|
6014
|
+
const key = ownerDispatchKey(scope, capability);
|
|
6015
|
+
if ((activeOwnerDispatches.get(key) ?? 0) !== 0) return { ok: false, reason: "children-active" };
|
|
6016
|
+
return mutateGovernor({ ...scope, identityProvider: null }, nowMs, (state, at) => {
|
|
6017
|
+
const group = state.debtGroups[capability.generation];
|
|
6018
|
+
if (!group) return { value: { status: "absent" } };
|
|
6019
|
+
if (group.nonce !== sealedNonce) return { ok: false, reason: "stale" };
|
|
6020
|
+
if (group.quiescent) return { value: { status: "acknowledged", nonce: group.nonce } };
|
|
6021
|
+
if (!group.ownerNonce || group.ownerNonce !== capability.ownerNonce ||
|
|
6022
|
+
!group.ownerBirth || !group.bootId ||
|
|
6023
|
+
group.ownerPid !== process.pid || group.ownerBirth !== coordinationProcessBirth() ||
|
|
6024
|
+
group.bootId !== coordinationBootId() ||
|
|
6025
|
+
Object.values(state.reservations).some((reservation) =>
|
|
6026
|
+
reservation.receipt?.generation === capability.generation)) {
|
|
6027
|
+
return { ok: false, reason: "quiescence-unverified" };
|
|
6028
|
+
}
|
|
6029
|
+
if (!makeDebtGroupQuiescent(state, group, at)) return { ok: false, reason: "accounting-overflow" };
|
|
6030
|
+
return { value: { status: "acknowledged", nonce: group.nonce } };
|
|
5164
6031
|
});
|
|
5165
6032
|
}
|
|
5166
6033
|
|
|
6034
|
+
function completeReservation(scope, reservationId, completion, nowMs) {
|
|
6035
|
+
const reservation = inspectGovernor({ ...scope, identityProvider: null }, nowMs);
|
|
6036
|
+
if (!reservation.ok) return reservation;
|
|
6037
|
+
const leaseId = reservation.value.reservations[reservationId]?.leaseId;
|
|
6038
|
+
if (!leaseId) return { ok: false, reason: "stale" };
|
|
6039
|
+
return settleReservationWithBudgetObservations(scope, leaseId, reservationId, {
|
|
6040
|
+
...completion,
|
|
6041
|
+
...(completion.actualCost ? { actualCosts: completion.actualCost } : {}),
|
|
6042
|
+
observations: [],
|
|
6043
|
+
}, nowMs);
|
|
6044
|
+
}
|
|
6045
|
+
|
|
6046
|
+
function terminalizeReservation(scope, capability, completion, nowMs) {
|
|
6047
|
+
if (!capability?.reservationId || !capability.leaseId) {
|
|
6048
|
+
return { ok: false, reason: "blocked", status: "blocked" };
|
|
6049
|
+
}
|
|
6050
|
+
const result = settleReservationWithBudgetObservations(
|
|
6051
|
+
{ ...scope, identityProvider: null }, capability.leaseId,
|
|
6052
|
+
capability.reservationId, { ...completion, capability }, nowMs,
|
|
6053
|
+
);
|
|
6054
|
+
if (result.ok) return { ok: true, value: { ...result.value, status: "settled" } };
|
|
6055
|
+
const status = ["busy", "unwritable"].includes(result.reason) ? "retryable"
|
|
6056
|
+
: result.reason === "stale" ? "compacted" : "blocked";
|
|
6057
|
+
return { ...result, status };
|
|
6058
|
+
}
|
|
6059
|
+
|
|
6060
|
+
const terminalizationBacklog = new Map();
|
|
6061
|
+
function terminalizationBacklogForResource(backlog, scopeHash, resource) {
|
|
6062
|
+
if (!scopeHash) return 0;
|
|
6063
|
+
let count = 0;
|
|
6064
|
+
for (const item of backlog.values()) {
|
|
6065
|
+
if (item.scope.hash !== scopeHash) continue;
|
|
6066
|
+
if (item.capability?.costs?.[resource] > 0 ||
|
|
6067
|
+
!RATE_RESOURCES.some((name) => item.capability?.costs?.[name] > 0)) count += 1;
|
|
6068
|
+
}
|
|
6069
|
+
return count;
|
|
6070
|
+
}
|
|
6071
|
+
function receiptCapabilityFromReservation(reservationId, reservation) {
|
|
6072
|
+
if (!reservation?.receipt || !reservation?.costs) return null;
|
|
6073
|
+
return {
|
|
6074
|
+
reservationId, leaseId: reservation.leaseId,
|
|
6075
|
+
scopeHash: reservation.receipt.scopeHash,
|
|
6076
|
+
ownerNonce: reservation.receipt.ownerNonce,
|
|
6077
|
+
generation: reservation.receipt.generation,
|
|
6078
|
+
deadline: reservation.receipt.deadline,
|
|
6079
|
+
costs: { ...reservation.costs },
|
|
6080
|
+
};
|
|
6081
|
+
}
|
|
6082
|
+
function queueTerminalization(scope, capability, completion, now = Date.now) {
|
|
6083
|
+
const frozen = { ...scope, identityProvider: null };
|
|
6084
|
+
const key = `${scope.hash}:${capability?.reservationId ?? "invalid"}`;
|
|
6085
|
+
const first = terminalizeReservation(frozen, capability, completion, now());
|
|
6086
|
+
if (first.ok || first.status !== "retryable") return first;
|
|
6087
|
+
if (terminalizationBacklog.has(key)) return first;
|
|
6088
|
+
if ([...terminalizationBacklog.keys()].filter((existing) =>
|
|
6089
|
+
existing.startsWith(`${scope.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
|
|
6090
|
+
return { ok: false, reason: "completion-capacity", status: "blocked" };
|
|
6091
|
+
}
|
|
6092
|
+
const item = { scope: frozen, capability, completion, now, attempt: 0, timer: null };
|
|
6093
|
+
const retry = () => {
|
|
6094
|
+
const result = terminalizeReservation(item.scope, item.capability, item.completion, item.now());
|
|
6095
|
+
if (result.ok || result.status !== "retryable") {
|
|
6096
|
+
terminalizationBacklog.delete(key);
|
|
6097
|
+
return;
|
|
6098
|
+
}
|
|
6099
|
+
const delays = [1000, 2000, 4000, 5000];
|
|
6100
|
+
const delay = delays[Math.min(item.attempt++, delays.length - 1)];
|
|
6101
|
+
item.timer = setTimeout(retry, delay + Math.floor(Math.random() * 100));
|
|
6102
|
+
item.timer.unref?.();
|
|
6103
|
+
};
|
|
6104
|
+
terminalizationBacklog.set(key, item);
|
|
6105
|
+
item.timer = setTimeout(retry, 1000);
|
|
6106
|
+
item.timer.unref?.();
|
|
6107
|
+
return first;
|
|
6108
|
+
}
|
|
6109
|
+
|
|
5167
6110
|
function settleReservationWithBudgetObservations(
|
|
5168
6111
|
scope,
|
|
5169
6112
|
leaseId,
|
|
@@ -5173,13 +6116,33 @@ function settleReservationWithBudgetObservations(
|
|
|
5173
6116
|
) {
|
|
5174
6117
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
5175
6118
|
const reservation = state.reservations[reservationId];
|
|
5176
|
-
|
|
6119
|
+
if (reservation?.status === "completed" && reservation.leaseId === leaseId &&
|
|
6120
|
+
(!completion?.capability ||
|
|
6121
|
+
completion.capability.ownerNonce === reservation.receipt?.ownerNonce &&
|
|
6122
|
+
completion.capability.generation === reservation.receipt?.generation)) {
|
|
6123
|
+
return { value: { status: "completed", actualCosts: reservation.actualCosts,
|
|
6124
|
+
accountedCosts: reservation.accountedCosts } };
|
|
6125
|
+
}
|
|
5177
6126
|
if (
|
|
5178
|
-
!
|
|
6127
|
+
!reservation ||
|
|
5179
6128
|
reservation.leaseId !== leaseId || reservation.status !== "started" ||
|
|
5180
6129
|
!GOVERNOR_OUTCOMES.has(completion?.outcome)
|
|
5181
6130
|
) return { ok: false, reason: "stale" };
|
|
5182
|
-
|
|
6131
|
+
if (completion.capability && (completion.capability.scopeHash !== scope.hash ||
|
|
6132
|
+
completion.capability.reservationId !== reservationId ||
|
|
6133
|
+
completion.capability.leaseId !== leaseId ||
|
|
6134
|
+
completion.capability.ownerNonce !== reservation.receipt?.ownerNonce ||
|
|
6135
|
+
completion.capability.generation !== reservation.receipt?.generation ||
|
|
6136
|
+
completion.capability.deadline !== reservation.receipt?.deadline)) {
|
|
6137
|
+
return { ok: false, reason: "receipt-invalid" };
|
|
6138
|
+
}
|
|
6139
|
+
const neverIssued = completion.capability && (
|
|
6140
|
+
completion.neverStarted === true && reservation.receipt.dispatches.length === 0 ||
|
|
6141
|
+
reservation.receipt.dispatches.length > 0 &&
|
|
6142
|
+
reservation.receipt.dispatches.every((dispatch) => dispatch.neverStarted));
|
|
6143
|
+
const outcome = neverIssued ? "measured-success" : completion.outcome;
|
|
6144
|
+
const measured = neverIssued ? { core: 0, graphql: 0 }
|
|
6145
|
+
: outcome === "measured-success"
|
|
5183
6146
|
? exactResourceCosts(completion.actualCosts ?? completion.actualCost)
|
|
5184
6147
|
: reservation.costs;
|
|
5185
6148
|
if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
|
|
@@ -5231,17 +6194,17 @@ function settleReservationWithBudgetObservations(
|
|
|
5231
6194
|
}
|
|
5232
6195
|
reservation.status = "completed";
|
|
5233
6196
|
reservation.completedAt = at;
|
|
5234
|
-
reservation.outcome =
|
|
6197
|
+
reservation.outcome = outcome;
|
|
5235
6198
|
reservation.actualCosts = { ...measured };
|
|
5236
6199
|
reservation.accountedCosts = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
5237
6200
|
resource,
|
|
5238
|
-
|
|
6201
|
+
outcome === "measured-success"
|
|
5239
6202
|
? Math.min(measured[resource], acceptedCosts[resource])
|
|
5240
6203
|
: 0,
|
|
5241
6204
|
]));
|
|
5242
6205
|
for (const resource of RATE_RESOURCES) {
|
|
5243
6206
|
const budget = state.budgets[resource];
|
|
5244
|
-
if (!budget ||
|
|
6207
|
+
if (!budget || outcome !== "measured-success") continue;
|
|
5245
6208
|
// accountedCosts removes the portion already present in an authoritative
|
|
5246
6209
|
// counter from the reservation residual. knownLocalUsed contains only
|
|
5247
6210
|
// that same portion when it arrived after the factor baseline. Their sum
|
|
@@ -5255,7 +6218,7 @@ function settleReservationWithBudgetObservations(
|
|
|
5255
6218
|
// loss proves nothing, so its worst case stays charged and its pacing stays
|
|
5256
6219
|
// spent -- refunding there would let a run of timeouts pace as though
|
|
5257
6220
|
// nothing had been sent.
|
|
5258
|
-
if (
|
|
6221
|
+
if (outcome === "measured-success") {
|
|
5259
6222
|
for (const resource of RATE_RESOURCES) {
|
|
5260
6223
|
returnPacingCredit(state, resource, reservation.costs[resource] - measured[resource], at);
|
|
5261
6224
|
}
|
|
@@ -5341,6 +6304,40 @@ function governorHealth(result, nowMs = Date.now()) {
|
|
|
5341
6304
|
};
|
|
5342
6305
|
}
|
|
5343
6306
|
|
|
6307
|
+
// Plain doctor must not call the identity coordinator: even its inspection
|
|
6308
|
+
// path may retire old registry attempts and persist the resulting registry.
|
|
6309
|
+
// A cached identity is useful only when it identifies one quota account for
|
|
6310
|
+
// the requested host. Multiple accounts cannot be resolved without probing.
|
|
6311
|
+
function doctorCachedQuotaScope(host, { root = identityRegistryRoot(), nowMs = Date.now() } = {}) {
|
|
6312
|
+
const registryPath = join(root, "registry.json");
|
|
6313
|
+
let registry;
|
|
6314
|
+
try {
|
|
6315
|
+
const file = lstatSync(registryPath);
|
|
6316
|
+
if (!file.isFile() || file.size > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "unsafe registry" };
|
|
6317
|
+
const parsed = JSON.parse(readFileSync(registryPath, "utf8"));
|
|
6318
|
+
registry = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
|
|
6319
|
+
} catch (error) {
|
|
6320
|
+
return { ok: false, reason: error?.code === "ENOENT" ? "cached identity missing" : "cached registry unreadable" };
|
|
6321
|
+
}
|
|
6322
|
+
if (!registry) return { ok: false, reason: "cached registry corrupt" };
|
|
6323
|
+
const quotas = new Set(Object.values(registry.identities)
|
|
6324
|
+
.filter((identity) => identity.host === host)
|
|
6325
|
+
.map((identity) => identity.quotaKey));
|
|
6326
|
+
if (quotas.size === 0) return { ok: false, reason: "cached identity missing" };
|
|
6327
|
+
if (quotas.size !== 1) return { ok: false, reason: "ambiguous cached identities" };
|
|
6328
|
+
const quotaKey = quotas.values().next().value;
|
|
6329
|
+
const path = join(root, `quota-${quotaKey}.json`);
|
|
6330
|
+
try {
|
|
6331
|
+
if (!lstatSync(path).isFile()) return { ok: false, reason: "unsafe quota path", path };
|
|
6332
|
+
} catch (error) {
|
|
6333
|
+
return { ok: false, reason: error?.code === "ENOENT" ? "cached quota missing" : "cached quota unreadable", path };
|
|
6334
|
+
}
|
|
6335
|
+
const result = readGovernorState(path, nowMs, { persistMigration: false });
|
|
6336
|
+
return result.ok && !result.missing
|
|
6337
|
+
? { ok: true, result, path, quotaKey }
|
|
6338
|
+
: { ok: false, reason: result.missing ? "cached quota missing" : result.reason, path };
|
|
6339
|
+
}
|
|
6340
|
+
|
|
5344
6341
|
async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
|
|
5345
6342
|
let result = run();
|
|
5346
6343
|
while (!result.ok && result.reason === "busy" && !signal?.aborted && now() < deadline) {
|
|
@@ -5470,7 +6467,8 @@ async function refreshResourceBudget(scope, leaseId, signal, resource, {
|
|
|
5470
6467
|
let budgets;
|
|
5471
6468
|
try {
|
|
5472
6469
|
try { onObserverCall?.(resource); } catch { /* diagnostics cannot block a budget observer */ }
|
|
5473
|
-
budgets = await requestIdentityStorage.run(
|
|
6470
|
+
budgets = await requestIdentityStorage.run({ ...scope,
|
|
6471
|
+
controlReceiptCapability: claim.value.receiptCapability }, () => readBudgets(signal, scope.host, {
|
|
5474
6472
|
resources: [resource],
|
|
5475
6473
|
coreEtag,
|
|
5476
6474
|
renewClaim: async () => {
|
|
@@ -6074,7 +7072,7 @@ const SSH_RECONNECT_STEPS_MS = Object.freeze([1_000, 2_000, 4_000, 8_000, 16_000
|
|
|
6074
7072
|
const SSH_STDERR_MAX_BYTES = 4_096;
|
|
6075
7073
|
const SSH_ALIAS_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$/;
|
|
6076
7074
|
const COLLECTOR_HOLD_REASONS = new Set([
|
|
6077
|
-
"observer", "primary", "secondary", "disconnected", "coordination", "shared-wait", "cache-only",
|
|
7075
|
+
"observer", "primary", "secondary", "disconnected", "app-auth", "coordination", "shared-wait", "cache-only",
|
|
6078
7076
|
]);
|
|
6079
7077
|
const COLLECTOR_ROW_KEYS = Object.freeze({
|
|
6080
7078
|
actions: ["databaseId", "displayTitle", "workflowName", "number", "headBranch", "status", "conclusion",
|
|
@@ -6592,7 +7590,7 @@ function persistGitHubAppIdentity(root, identity, now = Date.now()) {
|
|
|
6592
7590
|
const initialized = withGovernorLock(scope, () => {
|
|
6593
7591
|
const loaded = readIdentityQuotaState(state, scope, now);
|
|
6594
7592
|
if (!loaded.ok) return loaded;
|
|
6595
|
-
return loaded.missing ?
|
|
7593
|
+
return loaded.missing ? writeGovernorQuotaState(scope.path, loaded.value, now) : { ok: true };
|
|
6596
7594
|
});
|
|
6597
7595
|
if (!initialized.ok) return initialized;
|
|
6598
7596
|
state.identities[identity.credentialKey] = { host: identity.host, kind: identity.kind,
|
|
@@ -8403,7 +9401,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
|
|
|
8403
9401
|
if (!["changed", "unchanged"].includes(transition.kind)) {
|
|
8404
9402
|
const error = new Error("collector received unusable data");
|
|
8405
9403
|
error.requestMetrics = acquisitionRequestMetrics(result);
|
|
8406
|
-
error.uncertainReceipts = result.uncertainReceipts ?? [];
|
|
8407
9404
|
throw error;
|
|
8408
9405
|
}
|
|
8409
9406
|
const rows = transition.kind === "changed" ? transition.data : previous?.rows;
|
|
@@ -8435,7 +9432,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
|
|
|
8435
9432
|
hold: null,
|
|
8436
9433
|
capabilities: result.capabilities ?? previous?.capabilities ?? {},
|
|
8437
9434
|
requestMetrics: acquisitionRequestMetrics(result),
|
|
8438
|
-
uncertainReceipts: result.uncertainReceipts ?? [],
|
|
8439
9435
|
repositoryIdentity: result.repositoryIdentity,
|
|
8440
9436
|
meta,
|
|
8441
9437
|
securityNotes: resource === "security"
|
|
@@ -8616,6 +9612,7 @@ function createCollectorAcquisitionRuntime({
|
|
|
8616
9612
|
},
|
|
8617
9613
|
});
|
|
8618
9614
|
const reservationId = admitted.reservationId;
|
|
9615
|
+
const transportScope = { ...context.scope, receiptCapability: admitted.receiptCapability };
|
|
8619
9616
|
const governor = inspectGovernor(context.scope, now());
|
|
8620
9617
|
const started = await markStarted({
|
|
8621
9618
|
reservationId,
|
|
@@ -8623,8 +9620,9 @@ function createCollectorAcquisitionRuntime({
|
|
|
8623
9620
|
epochs: governor.ok ? governor.value.epochs : {},
|
|
8624
9621
|
});
|
|
8625
9622
|
if (!started.ok || started.value?.status !== "started") {
|
|
8626
|
-
|
|
8627
|
-
{ outcome: "rejected" }, now
|
|
9623
|
+
const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
|
|
9624
|
+
{ outcome: "rejected", neverStarted: true }, now);
|
|
9625
|
+
if (terminal.status === "blocked") throw new Error("collector receipt settlement blocked");
|
|
8628
9626
|
throw new Error("collector acquisition start failed");
|
|
8629
9627
|
}
|
|
8630
9628
|
const descriptor = tabForKey(resource);
|
|
@@ -8636,20 +9634,18 @@ function createCollectorAcquisitionRuntime({
|
|
|
8636
9634
|
signal,
|
|
8637
9635
|
AbortSignal.timeout(ACQUISITION_STARTED_DEADLINE_MS),
|
|
8638
9636
|
]);
|
|
8639
|
-
let result = await requestIdentityStorage.run(
|
|
9637
|
+
let result = await requestIdentityStorage.run(transportScope, () => descriptor.fetch({
|
|
8640
9638
|
signal: boundedSignal,
|
|
8641
9639
|
entities,
|
|
8642
9640
|
previousRaw: force ? null : snapshot?.raw ?? null,
|
|
8643
9641
|
force,
|
|
8644
9642
|
pages: demand.pages,
|
|
8645
|
-
governor: { scope:
|
|
9643
|
+
governor: { scope: transportScope, leaseId: context.leaseId },
|
|
8646
9644
|
...(securityPolicy ? { sources: securityPolicy.sources } : {}),
|
|
8647
9645
|
}));
|
|
8648
9646
|
if (securityPolicy) result = applyGitHubAppSecurityPolicy(result, securityPolicy);
|
|
8649
|
-
|
|
8650
|
-
|
|
8651
|
-
context.leaseId,
|
|
8652
|
-
reservationId,
|
|
9647
|
+
const terminal = queueTerminalization(
|
|
9648
|
+
context.scope, admitted.receiptCapability,
|
|
8653
9649
|
result?.measuredSuccess === false
|
|
8654
9650
|
? { outcome: "rejected", observations: result.observations ?? [] }
|
|
8655
9651
|
: {
|
|
@@ -8660,8 +9656,11 @@ function createCollectorAcquisitionRuntime({
|
|
|
8660
9656
|
},
|
|
8661
9657
|
observations: result.observations ?? [],
|
|
8662
9658
|
},
|
|
8663
|
-
now
|
|
9659
|
+
now,
|
|
8664
9660
|
);
|
|
9661
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
9662
|
+
throw new Error(`collector receipt settlement ${terminal.status}`);
|
|
9663
|
+
}
|
|
8665
9664
|
const completedAt = now();
|
|
8666
9665
|
const publication = collectorPublicationFromResult(
|
|
8667
9666
|
resource, result, snapshot, demand, completedAt, claim.state,
|
|
@@ -8677,10 +9676,11 @@ function createCollectorAcquisitionRuntime({
|
|
|
8677
9676
|
if (reconciliation !== null) publication.nextDueAt = completedAt + reconciliation;
|
|
8678
9677
|
return publication;
|
|
8679
9678
|
} catch (error) {
|
|
8680
|
-
|
|
8681
|
-
{
|
|
8682
|
-
|
|
8683
|
-
|
|
9679
|
+
const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
|
|
9680
|
+
{ outcome: governorOutcomeForError(error), observations: error.budgetObservations ?? [] }, now);
|
|
9681
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
9682
|
+
error.receiptSettlement = terminal.status;
|
|
9683
|
+
}
|
|
8684
9684
|
throw error;
|
|
8685
9685
|
}
|
|
8686
9686
|
}
|
|
@@ -8794,6 +9794,9 @@ function createCollectorAcquisitionRuntime({
|
|
|
8794
9794
|
item.nextDueAt = snapshot.nextDueAt;
|
|
8795
9795
|
if (Number.isSafeInteger(snapshot.generation)) item.lastDeliveredGeneration = snapshot.generation;
|
|
8796
9796
|
if (Number.isFinite(snapshot.lastSuccessAt)) item.lastDeliveredSuccessAt = snapshot.lastSuccessAt;
|
|
9797
|
+
if (publication && item.requestedAt !== null && snapshot.lastSuccessAt >= item.requestedAt) {
|
|
9798
|
+
item.requestedAt = null;
|
|
9799
|
+
}
|
|
8797
9800
|
item.onSnapshot(snapshot);
|
|
8798
9801
|
if (!publication || !Number.isSafeInteger(snapshot.generation)) return true;
|
|
8799
9802
|
for (const waiter of item.invalidations) {
|
|
@@ -8809,11 +9812,17 @@ function createCollectorAcquisitionRuntime({
|
|
|
8809
9812
|
}
|
|
8810
9813
|
|
|
8811
9814
|
function startPoll(item, { manual = false, force = false } = {}) {
|
|
9815
|
+
if (closed || item.closed) return null;
|
|
9816
|
+
if (manual && !demandEnabled(item.demand)) item.requestedAt ??= now();
|
|
9817
|
+
if (!item.id) {
|
|
9818
|
+
if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
|
|
9819
|
+
return startInitialize(item);
|
|
9820
|
+
}
|
|
8812
9821
|
if (item.pollPromise) {
|
|
8813
9822
|
if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
|
|
8814
9823
|
return item.pollPromise;
|
|
8815
9824
|
}
|
|
8816
|
-
if (
|
|
9825
|
+
if (!demandEnabled(item.demand) && item.requestedAt === null) return null;
|
|
8817
9826
|
item.pollPromise = poll(item, { manual, force }).finally(() => {
|
|
8818
9827
|
item.pollPromise = null;
|
|
8819
9828
|
if (item.promotionGeneration !== null) {
|
|
@@ -8832,7 +9841,7 @@ function createCollectorAcquisitionRuntime({
|
|
|
8832
9841
|
}
|
|
8833
9842
|
|
|
8834
9843
|
function schedule(item, delay, operation = () => startPoll(item)) {
|
|
8835
|
-
if (closed || item.closed || !demandEnabled(item.demand)) return;
|
|
9844
|
+
if (closed || item.closed || (!demandEnabled(item.demand) && item.requestedAt === null)) return;
|
|
8836
9845
|
if (item.timer !== null) clearTimeout_(item.timer);
|
|
8837
9846
|
item.timer = setTimeout_(() => {
|
|
8838
9847
|
item.timer = null;
|
|
@@ -8898,7 +9907,8 @@ function createCollectorAcquisitionRuntime({
|
|
|
8898
9907
|
() => engine.currentClaim(subscriptionId, claim), () => producer({
|
|
8899
9908
|
target: item.target,
|
|
8900
9909
|
resource: item.resource,
|
|
8901
|
-
demand: { ...ownership.value.demand, background: item.demand.background
|
|
9910
|
+
demand: { ...ownership.value.demand, background: item.demand.background,
|
|
9911
|
+
active: ownership.value.demand?.active || item.requestedAt !== null },
|
|
8902
9912
|
snapshot: ownership.value.snapshot,
|
|
8903
9913
|
claim: { ...claim, state: item },
|
|
8904
9914
|
provider: item.provider,
|
|
@@ -8919,12 +9929,11 @@ function createCollectorAcquisitionRuntime({
|
|
|
8919
9929
|
engine.refresh(subscriptionId, { claim, failure: {
|
|
8920
9930
|
hold: "disconnected",
|
|
8921
9931
|
requestMetrics: produced.requestMetrics ?? {},
|
|
8922
|
-
uncertainReceipts: produced.uncertainReceipts ?? [],
|
|
8923
9932
|
} });
|
|
8924
9933
|
engine.unsubscribe(subscriptionId);
|
|
8925
9934
|
item.id = null;
|
|
8926
9935
|
targetInitializations.delete(targetInitializationKey(item.target));
|
|
8927
|
-
item.onHold?.("
|
|
9936
|
+
item.onHold?.("app-auth");
|
|
8928
9937
|
retryIn = 1;
|
|
8929
9938
|
return;
|
|
8930
9939
|
}
|
|
@@ -8937,7 +9946,6 @@ function createCollectorAcquisitionRuntime({
|
|
|
8937
9946
|
engine.refresh(subscriptionId, { claim, failure: {
|
|
8938
9947
|
hold: "disconnected",
|
|
8939
9948
|
requestMetrics: produced.requestMetrics ?? {},
|
|
8940
|
-
uncertainReceipts: produced.uncertainReceipts ?? [],
|
|
8941
9949
|
} });
|
|
8942
9950
|
item.onHold?.("disconnected");
|
|
8943
9951
|
retryIn = item.demand.floorMs;
|
|
@@ -8960,10 +9968,12 @@ function createCollectorAcquisitionRuntime({
|
|
|
8960
9968
|
else if (claim) engine.refresh(subscriptionId, { claim, failure: {
|
|
8961
9969
|
hold,
|
|
8962
9970
|
requestMetrics: error?.requestMetrics ?? {},
|
|
8963
|
-
uncertainReceipts: error?.uncertainReceipts ?? [],
|
|
8964
9971
|
} });
|
|
8965
9972
|
else engine.setHold(subscriptionId, hold, { resource: item.resource, accessKey: item.identity.accessKey });
|
|
8966
|
-
item.onHold?.(
|
|
9973
|
+
item.onHold?.(item.provider?.definition?.type === "github-app" &&
|
|
9974
|
+
(error?.apiResponse?.status === 401 || error?.message === "GitHub App token unavailable" ||
|
|
9975
|
+
classify(error) === "auth-problem")
|
|
9976
|
+
? "app-auth" : hold);
|
|
8967
9977
|
try { onDiagnostic({ stage: "acquire", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
|
|
8968
9978
|
retryIn = error?.notStarted && Number.isFinite(error.retryAt)
|
|
8969
9979
|
? Math.max(50, error.retryAt - now()) : Math.min(1_000, item.demand.floorMs);
|
|
@@ -8979,13 +9989,16 @@ function createCollectorAcquisitionRuntime({
|
|
|
8979
9989
|
if (item.id && item.identity?.accessKey) {
|
|
8980
9990
|
engine.setHold(item.id, hold, { resource: item.resource, accessKey: item.identity.accessKey });
|
|
8981
9991
|
}
|
|
8982
|
-
item.onHold?.(
|
|
9992
|
+
item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
|
|
9993
|
+
error?.message === "GitHub App token unavailable" ? "app-auth" : hold);
|
|
8983
9994
|
try { onDiagnostic({ stage: "provider-refresh", reason: redact(shortErr(error)) }); } catch { /* isolated */ }
|
|
8984
9995
|
retryIn = Math.min(1_000, item.demand.floorMs);
|
|
8985
9996
|
} finally {
|
|
8986
9997
|
item.polling = false;
|
|
8987
9998
|
item.pollPhase = null;
|
|
8988
|
-
schedule(item, retryIn, item.id
|
|
9999
|
+
schedule(item, retryIn, item.id
|
|
10000
|
+
? () => startPoll(item, { manual: item.requestedAt !== null })
|
|
10001
|
+
: () => startInitialize(item));
|
|
8989
10002
|
}
|
|
8990
10003
|
}
|
|
8991
10004
|
|
|
@@ -9018,12 +10031,16 @@ function createCollectorAcquisitionRuntime({
|
|
|
9018
10031
|
if (subscription.value.snapshot && !accessReset) {
|
|
9019
10032
|
deliverCollectorSnapshot(item, subscription.value.snapshot, bindingRevision);
|
|
9020
10033
|
}
|
|
9021
|
-
if (demandEnabled(item.demand)) {
|
|
9022
|
-
|
|
10034
|
+
if (demandEnabled(item.demand) || item.requestedAt !== null) {
|
|
10035
|
+
const queued = item.queuedRefresh;
|
|
10036
|
+
item.queuedRefresh = null;
|
|
10037
|
+
await startPoll(item, { manual: item.invalidations.size > 0 || item.requestedAt !== null || Boolean(queued),
|
|
10038
|
+
force: queued?.force === true });
|
|
9023
10039
|
}
|
|
9024
10040
|
} catch (error) {
|
|
9025
10041
|
try { onDiagnostic({ stage: "initialize", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
|
|
9026
|
-
item.onHold?.("
|
|
10042
|
+
item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
|
|
10043
|
+
error?.message === "GitHub App token unavailable" ? "app-auth" : "disconnected");
|
|
9027
10044
|
schedule(item, 1_000, () => startInitialize(item));
|
|
9028
10045
|
}
|
|
9029
10046
|
}
|
|
@@ -9040,7 +10057,7 @@ function createCollectorAcquisitionRuntime({
|
|
|
9040
10057
|
if (!normalizedDemand) throw new Error("invalid collector demand");
|
|
9041
10058
|
const item = { target, resource, demand: normalizedDemand, onSnapshot, onHold, id: null, identity: null,
|
|
9042
10059
|
provider: null, polling: false, pollPhase: null, pollPromise: null, controller: null,
|
|
9043
|
-
initializePromise: null, closed: false, timer: null, queuedRefresh: null,
|
|
10060
|
+
initializePromise: null, closed: false, timer: null, queuedRefresh: null, requestedAt: null,
|
|
9044
10061
|
bindingRevision: 0, invalidations: new Set(), lastDeliveredGeneration: null,
|
|
9045
10062
|
lastDeliveredSuccessAt: null, promotionGeneration: null, promotionAt: null };
|
|
9046
10063
|
items.add(item);
|
|
@@ -9349,7 +10366,14 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9349
10366
|
},
|
|
9350
10367
|
});
|
|
9351
10368
|
const write = (message) => {
|
|
9352
|
-
if (welcomed
|
|
10369
|
+
if (!welcomed || !socket?.writable) return false;
|
|
10370
|
+
socket.write(encodeCollectorFrame(message));
|
|
10371
|
+
return true;
|
|
10372
|
+
};
|
|
10373
|
+
const flushRefresh = (item) => {
|
|
10374
|
+
if (item.pendingRefresh !== null && write({ type: "refresh", id: item.id, force: item.pendingRefresh })) {
|
|
10375
|
+
item.pendingRefresh = null;
|
|
10376
|
+
}
|
|
9353
10377
|
};
|
|
9354
10378
|
const resubscribe = (resetGenerations) => {
|
|
9355
10379
|
for (const item of subscriptions.values()) {
|
|
@@ -9359,6 +10383,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9359
10383
|
}
|
|
9360
10384
|
write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
|
|
9361
10385
|
resource: item.resource, demand: item.demand });
|
|
10386
|
+
flushRefresh(item);
|
|
9362
10387
|
}
|
|
9363
10388
|
};
|
|
9364
10389
|
const connect = () => {
|
|
@@ -9449,7 +10474,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9449
10474
|
return { ok: false, reason: "invalid" };
|
|
9450
10475
|
}
|
|
9451
10476
|
const item = { ...subscription,
|
|
9452
|
-
onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1,
|
|
10477
|
+
onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1, pendingRefresh: null,
|
|
9453
10478
|
sourceAge: createCollectorSourceAgeTracker({ now, monotonicNow, checkpoint: sourceCheckpoint }) };
|
|
9454
10479
|
subscriptions.set(id, item);
|
|
9455
10480
|
if (welcomed) write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
|
|
@@ -9467,7 +10492,10 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9467
10492
|
write({ type: "demand", id, demand: normalized });
|
|
9468
10493
|
return { ok: true, changed: true };
|
|
9469
10494
|
},
|
|
9470
|
-
refresh(force = false) {
|
|
10495
|
+
refresh(force = false) {
|
|
10496
|
+
item.pendingRefresh = force === true || item.pendingRefresh === true;
|
|
10497
|
+
flushRefresh(item);
|
|
10498
|
+
},
|
|
9471
10499
|
inspect() { write({ type: "inspect", id }); },
|
|
9472
10500
|
close() { write({ type: "unsubscribe", id }); subscriptions.delete(id); assembler.cancel(id); },
|
|
9473
10501
|
} };
|
|
@@ -9508,6 +10536,7 @@ function createSshCollectorClient({ alias, createTransport = null, random = Math
|
|
|
9508
10536
|
}
|
|
9509
10537
|
|
|
9510
10538
|
function collectorDisplayDecision({ connected, hasSnapshot, hold = null }) {
|
|
10539
|
+
if (hold === "app-auth") return { disconnected: true, reason: "app-auth" };
|
|
9511
10540
|
if (!connected || hold === "disconnected") return { disconnected: true };
|
|
9512
10541
|
if (hold === "shared-wait") return { mode: "waiting", waitCause: "shared-lane" };
|
|
9513
10542
|
if (["observer", "primary", "secondary", "coordination"].includes(hold)) {
|
|
@@ -9947,8 +10976,10 @@ const ACQUISITION_METRIC_KEYS = [
|
|
|
9947
10976
|
const ACQUISITION_HOLD_REASONS = new Set([
|
|
9948
10977
|
"observer", "primary", "secondary", "disconnected", "coordination",
|
|
9949
10978
|
]);
|
|
9950
|
-
const ACQUISITION_STORE_VERSION =
|
|
10979
|
+
const ACQUISITION_STORE_VERSION = 2;
|
|
9951
10980
|
const ACQUISITION_MAX_UNCERTAIN_RECEIPTS = 1024;
|
|
10981
|
+
const ACQUISITION_MAX_QUOTA_PROJECTIONS = 128;
|
|
10982
|
+
const ACQUISITION_QUOTA_PROJECTION_TTL_MS = 60_000;
|
|
9952
10983
|
|
|
9953
10984
|
function normalizeAcquisitionDiagnosticMetadata(raw, path) {
|
|
9954
10985
|
const normalized = normalizeAcquisitionStore(raw);
|
|
@@ -10098,16 +11129,23 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10098
11129
|
if (!admitted?.ok || admitted.value.status !== "started") {
|
|
10099
11130
|
return { index, skipped: { ...skippedDoctorProbe(name, args, admitted), ...(document ? { document } : {}) } };
|
|
10100
11131
|
}
|
|
10101
|
-
return { index, name, args, operation, input, document,
|
|
11132
|
+
return { index, name, args, operation, input, document,
|
|
11133
|
+
reservationId: admitted.value.reservationId,
|
|
11134
|
+
receiptCapability: admitted.value.receiptCapability };
|
|
10102
11135
|
});
|
|
10103
11136
|
results = admittedProbes.map((item) => item.skipped ?? null);
|
|
10104
11137
|
const runnable = admittedProbes.filter((item) => !item.skipped);
|
|
10105
11138
|
const settled = await mapAllSettledBounded(runnable, 4, async (item) => {
|
|
10106
|
-
const result = await requestIdentityStorage.run(scope,
|
|
11139
|
+
const result = await requestIdentityStorage.run({ ...scope,
|
|
11140
|
+
receiptCapability: item.receiptCapability }, () => probe(item.name, item.args, item.operation, item.input));
|
|
10107
11141
|
if (item.document) result.document = item.document;
|
|
10108
|
-
|
|
11142
|
+
const terminal = queueTerminalization(scope, item.receiptCapability, result.failed
|
|
10109
11143
|
? { outcome: "rejected" }
|
|
10110
|
-
: { outcome: "measured-success",
|
|
11144
|
+
: { outcome: "measured-success", actualCosts: operationCost(item.operation) });
|
|
11145
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
11146
|
+
result.failed = true;
|
|
11147
|
+
result.stderr = `receipt settlement ${terminal.status}`;
|
|
11148
|
+
}
|
|
10111
11149
|
return result;
|
|
10112
11150
|
});
|
|
10113
11151
|
settled.forEach((outcome, index) => {
|
|
@@ -10128,9 +11166,32 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10128
11166
|
governorResult = inspectGovernor(cleanup, Date.now());
|
|
10129
11167
|
}
|
|
10130
11168
|
}
|
|
11169
|
+
const cachedQuota = !probeEndpoints ? doctorCachedQuotaScope(effectiveHost) : null;
|
|
10131
11170
|
const governor = probeEndpoints
|
|
10132
11171
|
? governorHealth(governorResult, Date.now())
|
|
10133
|
-
:
|
|
11172
|
+
: cachedQuota?.ok ? governorHealth(cachedQuota.result, Date.now())
|
|
11173
|
+
: { status: `unavailable (${cachedQuota?.reason ?? "cached scope unavailable"})`, leases: 0, resources: {} };
|
|
11174
|
+
const diagnosticScope = probeEndpoints
|
|
11175
|
+
? scopeResult?.ok ? scopeResult.value : null
|
|
11176
|
+
: cachedQuota?.ok ? { path: cachedQuota.path } : null;
|
|
11177
|
+
const recoveryJournal = diagnosticScope ? readRecoveryTransitions(diagnosticScope) : null;
|
|
11178
|
+
const recoveryDetails = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
11179
|
+
diagnosticScope ? recoveryLedgerSummary(diagnosticScope, resource) : null]));
|
|
11180
|
+
const currentRecovery = RATE_RESOURCES.map((resource) => recoveryJournal?.current[resource])
|
|
11181
|
+
.filter(Boolean).sort((left, right) => right.at - left.at)[0] ?? null;
|
|
11182
|
+
const lastFailedRecovery = [...(recoveryJournal?.entries ?? [])].reverse()
|
|
11183
|
+
.find((entry) => entry.code !== "clear") ?? null;
|
|
11184
|
+
const retryDates = RATE_RESOURCES.flatMap((resource) => {
|
|
11185
|
+
const active = recoveryJournal?.current[resource];
|
|
11186
|
+
const observer = (probeEndpoints ? governorResult : cachedQuota?.result)?.value?.observers[resource];
|
|
11187
|
+
return [active?.retryAt, observer?.outcome === "failed" ? observer.nextAt : null];
|
|
11188
|
+
}).filter((value) => Number.isFinite(value) && value > Date.now());
|
|
11189
|
+
const nextRecoveryAt = retryDates.length > 0 ? Math.min(...retryDates) : null;
|
|
11190
|
+
let recoveryReason = "unavailable";
|
|
11191
|
+
if (currentRecovery) recoveryReason = `${currentRecovery.resource}: ${currentRecovery.code}`;
|
|
11192
|
+
else if (recoveryJournal?.status === "corrupt") recoveryReason = "diagnostic journal corrupt";
|
|
11193
|
+
else if (recoveryJournal?.status === "unavailable") recoveryReason = "diagnostic journal unavailable";
|
|
11194
|
+
else if (["healthy", "missing"].includes(recoveryJournal?.status)) recoveryReason = "none recorded";
|
|
10134
11195
|
const diagnosticSource = runtime.connect === "local"
|
|
10135
11196
|
? "local collector"
|
|
10136
11197
|
: runtime.connect?.startsWith("ssh:") ? "SSH collector" : "standalone";
|
|
@@ -10175,6 +11236,10 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10175
11236
|
),
|
|
10176
11237
|
"",
|
|
10177
11238
|
...section("API governor"),
|
|
11239
|
+
field("cached scope", probeEndpoints ? "verified by probe" : cachedQuota?.ok ? "read-only cached identity" : `unavailable (${cachedQuota?.reason ?? "unknown"})`),
|
|
11240
|
+
...(cachedQuota?.path && !cachedQuota.ok
|
|
11241
|
+
? [field("state file", `${join(dirname(cachedQuota.path), "quota-[redacted].json")} (quota fingerprint withheld)`)]
|
|
11242
|
+
: []),
|
|
10178
11243
|
field("status", governor.status),
|
|
10179
11244
|
field("live leases", governor.leases),
|
|
10180
11245
|
...RATE_RESOURCES.map((resource) => {
|
|
@@ -10186,6 +11251,21 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10186
11251
|
: "unavailable",
|
|
10187
11252
|
);
|
|
10188
11253
|
}),
|
|
11254
|
+
field("recovery reason", recoveryReason),
|
|
11255
|
+
field("last failed transition", lastFailedRecovery?.code ?? "unavailable"),
|
|
11256
|
+
...RATE_RESOURCES.flatMap((resource) => [
|
|
11257
|
+
field(`${resource} receipt units`, recoveryDetails[resource]?.receiptUnits ?? "unavailable"),
|
|
11258
|
+
field(`${resource} receipt count`, recoveryDetails[resource]?.receiptCount ?? "unavailable"),
|
|
11259
|
+
field(`${resource} debt units`, recoveryDetails[resource]?.debtUnits ?? "unavailable"),
|
|
11260
|
+
field(`${resource} debt count`, recoveryDetails[resource]?.debtCount ?? "unavailable"),
|
|
11261
|
+
field(`${resource} oldest unresolved`, recoveryDetails[resource]?.oldestUnresolvedAt == null
|
|
11262
|
+
? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].oldestUnresolvedAt))),
|
|
11263
|
+
field(`${resource} observer age`, recoveryDetails[resource]?.observerAt == null
|
|
11264
|
+
? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].observerAt))),
|
|
11265
|
+
]),
|
|
11266
|
+
field("next retry", nextRecoveryAt == null ? "unavailable" : new Date(nextRecoveryAt).toISOString()),
|
|
11267
|
+
field("terminalization backlog", currentRecovery?.terminalizationBacklog == null
|
|
11268
|
+
? "unavailable" : `${currentRecovery.terminalizationBacklog} (last recorded)`),
|
|
10189
11269
|
"",
|
|
10190
11270
|
...section("Acquisition metrics"),
|
|
10191
11271
|
field("source", acquisitionDiagnostic?.source ?? "standalone"),
|
|
@@ -10202,7 +11282,8 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10202
11282
|
field("failed requests", acquisitionDiagnostic?.metrics.failedRequests ?? 0),
|
|
10203
11283
|
field("REST actual 200/304", `${acquisitionDiagnostic?.metrics.rest200 ?? 0}/${acquisitionDiagnostic?.metrics.rest304 ?? 0}`),
|
|
10204
11284
|
field("proven actual units", `${acquisitionDiagnostic?.metrics.coreUnits ?? 0} core + ${acquisitionDiagnostic?.metrics.graphqlUnits ?? 0} GraphQL`),
|
|
10205
|
-
field("conservative outstanding",
|
|
11285
|
+
field("conservative outstanding", acquisitionOutstandingText(acquisitionDiagnostic)),
|
|
11286
|
+
field("legacy unverified", `${acquisitionDiagnostic?.legacyUnverified?.coreUnits ?? 0} core + ${acquisitionDiagnostic?.legacyUnverified?.graphqlUnits ?? 0} GraphQL (diagnostic only)`),
|
|
10206
11287
|
field("observer calls", acquisitionDiagnostic?.metrics.observerCalls ?? 0),
|
|
10207
11288
|
field("cache/followers", `${acquisitionDiagnostic?.metrics.cacheHits ?? 0}/${acquisitionDiagnostic?.metrics.joinedFollowers ?? 0}`),
|
|
10208
11289
|
field("queue wait (measured)", `${acquisitionDiagnostic?.metrics.queueWaitMs ?? 0}ms`),
|
|
@@ -10321,7 +11402,7 @@ function parseRepoTarget(value) {
|
|
|
10321
11402
|
// in-flight guard absorbs every other one and the effective rate is whatever
|
|
10322
11403
|
// `gh` can sustain -- the requested interval silently stops being real. Clamping
|
|
10323
11404
|
// with a stated minimum is honest where silently accepting it would not be.
|
|
10324
|
-
const BACKGROUND_MODES = ["all", "off"];
|
|
11405
|
+
const BACKGROUND_MODES = ["all", "off", "on-demand"];
|
|
10325
11406
|
const MIN_REFRESH_SECONDS = 2;
|
|
10326
11407
|
const MAX_REFRESH_SECONDS = 3600;
|
|
10327
11408
|
|
|
@@ -10551,7 +11632,11 @@ Options:
|
|
|
10551
11632
|
and running Actions are checked every ${POLL_ACTIVE_CI_MS / 1000}s.
|
|
10552
11633
|
Safe shared grants may run later.
|
|
10553
11634
|
--tab <name> Tab to start on: ${TAB_KEYS.join(", ")}.
|
|
10554
|
-
--background <mode> ${BACKGROUND_MODES.join(" or ")} (default
|
|
11635
|
+
--background <mode> ${BACKGROUND_MODES.join(" or ")} (default on-demand).
|
|
11636
|
+
on-demand watches active Actions; other tabs refresh
|
|
11637
|
+
when opened or with r, then keep cached results.
|
|
11638
|
+
all polls every tab; off polls the selected tab.
|
|
11639
|
+
\`off\` never requests data
|
|
10555
11640
|
for a tab you are not looking at; its count stays at
|
|
10556
11641
|
the last known value and visibly ages.
|
|
10557
11642
|
--verbose Write one line per gh invocation to stderr. stderr
|
|
@@ -10579,9 +11664,11 @@ Run it from inside a locally cloned GitHub repository; the repo is inferred
|
|
|
10579
11664
|
from the git remote, the same way \`gh\` does it. Requires the \`gh\` CLI
|
|
10580
11665
|
(2.20 or newer), authenticated via \`gh auth login\`.
|
|
10581
11666
|
|
|
10582
|
-
|
|
11667
|
+
By default, active Actions is considered every ${REFRESH_MS / 1000}s while its
|
|
10583
11668
|
content is changing, then every ${POLL_QUIET_MS.actions / 1000}s once two checks in a row come back
|
|
10584
|
-
unchanged
|
|
11669
|
+
unchanged. Secondary tabs check on opening or r and retain cached results.
|
|
11670
|
+
With --background all, inactive tabs are considered about every
|
|
11671
|
+
${Math.max(REFRESH_MS * BACKGROUND_EVERY, POLL_BACKGROUND_MS.actions) / 1000}s (Security is slower). Quota-consuming
|
|
10585
11672
|
calls still need a shared, resource-specific grant that preserves a hard reserve.
|
|
10586
11673
|
Scheduled checks and manual refresh do not bypass that safety check: neither
|
|
10587
11674
|
refresh key can spend past the reserve.
|
|
@@ -12326,6 +13413,8 @@ const ACQUISITION_MAX_ENTITY_BYTES = 1024 * 1024;
|
|
|
12326
13413
|
const ACQUISITION_MAX_ENTITIES = 512;
|
|
12327
13414
|
const ACQUISITION_MAX_LIVE_TARGETS = 32;
|
|
12328
13415
|
const ACQUISITION_MAX_SUBSCRIPTIONS = 128;
|
|
13416
|
+
const ACQUISITION_READ_MAX_ATTEMPTS = 24;
|
|
13417
|
+
const ACQUISITION_READ_MAX_MS = 1_500;
|
|
12329
13418
|
let acquisitionTempSequence = 0;
|
|
12330
13419
|
|
|
12331
13420
|
function acquisitionStorePath(options = {}) {
|
|
@@ -12337,7 +13426,8 @@ function emptyAcquisitionStore() {
|
|
|
12337
13426
|
version: ACQUISITION_STORE_VERSION,
|
|
12338
13427
|
producerEpoch: governorId(),
|
|
12339
13428
|
metrics: emptyAcquisitionMetrics(),
|
|
12340
|
-
|
|
13429
|
+
quotaProjections: {},
|
|
13430
|
+
legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
|
|
12341
13431
|
subscriptions: {},
|
|
12342
13432
|
queries: {},
|
|
12343
13433
|
aliases: {},
|
|
@@ -12365,32 +13455,6 @@ function acquisitionMetricState(state) {
|
|
|
12365
13455
|
return state.metrics;
|
|
12366
13456
|
}
|
|
12367
13457
|
|
|
12368
|
-
function removeAcquisitionReceipt(state, id) {
|
|
12369
|
-
const receipt = state.uncertainReceipts?.[id];
|
|
12370
|
-
if (!receipt) return false;
|
|
12371
|
-
const metrics = acquisitionMetricState(state);
|
|
12372
|
-
const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
|
|
12373
|
-
metrics[key] = Math.max(0, metrics[key] - receipt.units);
|
|
12374
|
-
delete state.uncertainReceipts[id];
|
|
12375
|
-
for (const record of Object.values(state.queries ?? {})) {
|
|
12376
|
-
if (record.claim?.receiptIds?.includes(id)) {
|
|
12377
|
-
record.claim.receiptIds = record.claim.receiptIds.filter((receiptId) => receiptId !== id);
|
|
12378
|
-
}
|
|
12379
|
-
}
|
|
12380
|
-
return true;
|
|
12381
|
-
}
|
|
12382
|
-
|
|
12383
|
-
function addAcquisitionReceipt(state, receipt) {
|
|
12384
|
-
state.uncertainReceipts ??= {};
|
|
12385
|
-
if (state.uncertainReceipts[receipt.id]) return true;
|
|
12386
|
-
if (Object.keys(state.uncertainReceipts).length >= ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return false;
|
|
12387
|
-
state.uncertainReceipts[receipt.id] = receipt;
|
|
12388
|
-
const metrics = acquisitionMetricState(state);
|
|
12389
|
-
const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
|
|
12390
|
-
metrics[key] += receipt.units;
|
|
12391
|
-
return true;
|
|
12392
|
-
}
|
|
12393
|
-
|
|
12394
13458
|
function normalizeAcquisitionReceipt(raw, expectedId = raw?.id) {
|
|
12395
13459
|
if (!isRecord(raw) || raw.id !== expectedId || typeof raw.id !== "string" || raw.id.length > 240 ||
|
|
12396
13460
|
typeof raw.reservationId !== "string" || raw.reservationId.length < 1 ||
|
|
@@ -12415,63 +13479,6 @@ function addAcquisitionRequestMetrics(state, delta) {
|
|
|
12415
13479
|
return addAcquisitionMetrics(state, observed);
|
|
12416
13480
|
}
|
|
12417
13481
|
|
|
12418
|
-
function releaseAcquisitionUncertainty(state, record) {
|
|
12419
|
-
if (!record?.claim?.started) return;
|
|
12420
|
-
const receiptIds = record.claim.receiptIds ?? [];
|
|
12421
|
-
if (Object.hasOwn(record.claim, "receiptIds")) {
|
|
12422
|
-
for (const id of receiptIds) removeAcquisitionReceipt(state, id);
|
|
12423
|
-
return;
|
|
12424
|
-
}
|
|
12425
|
-
const costs = tabRequestCost(record.query.resource);
|
|
12426
|
-
const metrics = acquisitionMetricState(state);
|
|
12427
|
-
metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - costs.core);
|
|
12428
|
-
metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - costs.graphql);
|
|
12429
|
-
}
|
|
12430
|
-
|
|
12431
|
-
function releaseKnownAcquisitionUncertainty(state, record, requestMetrics) {
|
|
12432
|
-
if (!record?.claim?.started) return;
|
|
12433
|
-
const receiptIds = record.claim.receiptIds ?? [];
|
|
12434
|
-
if (Object.hasOwn(record.claim, "receiptIds")) {
|
|
12435
|
-
for (const resource of RATE_RESOURCES) {
|
|
12436
|
-
const metricKey = resource === "core" ? "coreUnits" : "graphqlUnits";
|
|
12437
|
-
const uncertainKey = resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
|
|
12438
|
-
const resourceReceipts = receiptIds
|
|
12439
|
-
.map((id) => state.uncertainReceipts?.[id])
|
|
12440
|
-
.filter((receipt) => receipt?.resource === resource);
|
|
12441
|
-
if (Object.hasOwn(requestMetrics, uncertainKey)) {
|
|
12442
|
-
let remaining = requestMetrics[uncertainKey];
|
|
12443
|
-
for (const receipt of resourceReceipts) {
|
|
12444
|
-
const retained = Math.min(receipt.units, remaining);
|
|
12445
|
-
const released = receipt.units - retained;
|
|
12446
|
-
remaining -= retained;
|
|
12447
|
-
if (retained === 0) removeAcquisitionReceipt(state, receipt.id);
|
|
12448
|
-
else if (released > 0) {
|
|
12449
|
-
receipt.units = retained;
|
|
12450
|
-
acquisitionMetricState(state)[uncertainKey] -= released;
|
|
12451
|
-
}
|
|
12452
|
-
}
|
|
12453
|
-
continue;
|
|
12454
|
-
}
|
|
12455
|
-
if (Object.hasOwn(requestMetrics, metricKey)) {
|
|
12456
|
-
for (const receipt of resourceReceipts) removeAcquisitionReceipt(state, receipt.id);
|
|
12457
|
-
}
|
|
12458
|
-
}
|
|
12459
|
-
return;
|
|
12460
|
-
}
|
|
12461
|
-
const declared = tabRequestCost(record.query.resource);
|
|
12462
|
-
const metrics = acquisitionMetricState(state);
|
|
12463
|
-
if (Object.hasOwn(requestMetrics, "uncertainCoreUnits")) {
|
|
12464
|
-
metrics.uncertainCoreUnits = Math.min(declared.core, requestMetrics.uncertainCoreUnits);
|
|
12465
|
-
} else if (Object.hasOwn(requestMetrics, "coreUnits")) {
|
|
12466
|
-
metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - declared.core);
|
|
12467
|
-
}
|
|
12468
|
-
if (Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
|
|
12469
|
-
metrics.uncertainGraphqlUnits = Math.min(declared.graphql, requestMetrics.uncertainGraphqlUnits);
|
|
12470
|
-
} else if (Object.hasOwn(requestMetrics, "graphqlUnits")) {
|
|
12471
|
-
metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - declared.graphql);
|
|
12472
|
-
}
|
|
12473
|
-
}
|
|
12474
|
-
|
|
12475
13482
|
function settleAcquisitionWaiters(state, queryKey, generation, at) {
|
|
12476
13483
|
let queueWaitMs = 0;
|
|
12477
13484
|
for (const subscription of Object.values(state.subscriptions)) {
|
|
@@ -12490,8 +13497,6 @@ function settleInvalidAcquisitionPublication(state, record, claim, requestMetric
|
|
|
12490
13497
|
...normalized,
|
|
12491
13498
|
failedRequests: Math.max(1, normalized.failedRequests ?? 0),
|
|
12492
13499
|
});
|
|
12493
|
-
releaseKnownAcquisitionUncertainty(state, record, normalized);
|
|
12494
|
-
for (const receipt of receipts ?? []) addAcquisitionReceipt(state, receipt);
|
|
12495
13500
|
settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, at);
|
|
12496
13501
|
record.claim = null;
|
|
12497
13502
|
record.hold = { reason: "primary", at };
|
|
@@ -12528,9 +13533,18 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
|
|
|
12528
13533
|
subscriptionsByQuery.set(subscription.queryKey, grouped);
|
|
12529
13534
|
}
|
|
12530
13535
|
const activeQueryKeys = new Set(subscriptions.map((subscription) => subscription.queryKey));
|
|
13536
|
+
const outstandingQuota = Object.entries(state?.quotaProjections ?? {}).map(([scopeHash, projection]) => {
|
|
13537
|
+
const current = projection.status === "current" && projection.at <= nowMs &&
|
|
13538
|
+
nowMs - projection.at <= ACQUISITION_QUOTA_PROJECTION_TTL_MS;
|
|
13539
|
+
return { scopeHash, revision: projection.revision,
|
|
13540
|
+
status: current ? "current" : "unavailable", units: current ? projection.units : null,
|
|
13541
|
+
at: projection.at, accessKeys: projection.accessKeys };
|
|
13542
|
+
});
|
|
12531
13543
|
const queries = Object.values(state?.queries ?? {}).map((record) => {
|
|
12532
13544
|
const snapshot = record.snapshot;
|
|
12533
13545
|
const consumers = subscriptionsByQuery.get(record.query.queryKey) ?? [];
|
|
13546
|
+
const projection = outstandingQuota.find((item) =>
|
|
13547
|
+
item.accessKeys.includes(record.query.accessKey) && item.status === "current");
|
|
12534
13548
|
return {
|
|
12535
13549
|
resource: record.query.resource,
|
|
12536
13550
|
lastSuccessAt: snapshot?.lastSuccessAt ?? null,
|
|
@@ -12543,8 +13557,9 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
|
|
|
12543
13557
|
graphql: state?.metrics?.graphqlUnits ?? 0,
|
|
12544
13558
|
},
|
|
12545
13559
|
uncertainCost: {
|
|
12546
|
-
|
|
12547
|
-
|
|
13560
|
+
status: projection ? "current" : "unavailable",
|
|
13561
|
+
core: projection?.units.core ?? null,
|
|
13562
|
+
graphql: projection?.units.graphql ?? null,
|
|
12548
13563
|
},
|
|
12549
13564
|
coalescedConsumers: consumers.length,
|
|
12550
13565
|
};
|
|
@@ -12556,12 +13571,31 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
|
|
|
12556
13571
|
activeQueries: activeQueryKeys.size,
|
|
12557
13572
|
activeSubscribers: subscriptions.length,
|
|
12558
13573
|
metrics: normalizeAcquisitionMetrics(state?.metrics) ?? emptyAcquisitionMetrics(),
|
|
13574
|
+
outstandingQuota: outstandingQuota.map(({ accessKeys: _accessKeys, ...projection }) => projection),
|
|
13575
|
+
legacyUnverified: state?.legacyUnverified ?? { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
|
|
12559
13576
|
queries,
|
|
12560
13577
|
};
|
|
12561
13578
|
}
|
|
12562
13579
|
|
|
13580
|
+
function acquisitionOutstandingText(diagnostic) {
|
|
13581
|
+
const scopes = diagnostic?.outstandingQuota ?? [];
|
|
13582
|
+
if (scopes.length === 0 || scopes.some((scope) => scope.status !== "current")) {
|
|
13583
|
+
return "unavailable (quota projection pending)";
|
|
13584
|
+
}
|
|
13585
|
+
const units = scopes.reduce((total, scope) => ({
|
|
13586
|
+
core: total.core + scope.units.core,
|
|
13587
|
+
graphql: total.graphql + scope.units.graphql,
|
|
13588
|
+
}), { core: 0, graphql: 0 });
|
|
13589
|
+
return Number.isSafeInteger(units.core) && Number.isSafeInteger(units.graphql)
|
|
13590
|
+
? `${units.core} core + ${units.graphql} GraphQL across ${scopes.length} quota scopes`
|
|
13591
|
+
: "unavailable (accounting overflow)";
|
|
13592
|
+
}
|
|
13593
|
+
|
|
12563
13594
|
function acquisitionFailureHold(error) {
|
|
12564
13595
|
if (typeof error?.providerCapability === "string") return "observer";
|
|
13596
|
+
if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) {
|
|
13597
|
+
return HTTP_PERMIT_FAILURE_HOLDS[error.coordinationReason];
|
|
13598
|
+
}
|
|
12565
13599
|
if (SECONDARY_LIMIT_PATTERN.test(errText(error))) return "secondary";
|
|
12566
13600
|
if (isRateLimited(error) || /API budget paused/i.test(errText(error))) return "primary";
|
|
12567
13601
|
const attemptedHttp = Number.isSafeInteger(error?.requestMetrics?.httpRequests) &&
|
|
@@ -12889,32 +13923,58 @@ function prepareAcquisitionSnapshot(queryKey, snapshot) {
|
|
|
12889
13923
|
}
|
|
12890
13924
|
|
|
12891
13925
|
function normalizeAcquisitionStore(raw) {
|
|
12892
|
-
if (!isRecord(raw) || raw.version
|
|
13926
|
+
if (!isRecord(raw) || ![1, ACQUISITION_STORE_VERSION].includes(raw.version) || !validGovernorId(raw.producerEpoch) ||
|
|
12893
13927
|
!isRecord(raw.subscriptions) || !isRecord(raw.queries) || !isRecord(raw.aliases)) return null;
|
|
13928
|
+
const legacy = raw.version === 1;
|
|
12894
13929
|
const state = {
|
|
12895
13930
|
version: ACQUISITION_STORE_VERSION,
|
|
12896
|
-
producerEpoch: raw.producerEpoch,
|
|
13931
|
+
producerEpoch: legacy ? governorId() : raw.producerEpoch,
|
|
12897
13932
|
metrics: normalizeAcquisitionMetrics(raw.metrics),
|
|
12898
|
-
|
|
13933
|
+
quotaProjections: {},
|
|
13934
|
+
legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
|
|
12899
13935
|
subscriptions: {},
|
|
12900
13936
|
queries: {},
|
|
12901
13937
|
aliases: {},
|
|
12902
13938
|
};
|
|
12903
13939
|
if (state.metrics === null) return null;
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
const
|
|
12908
|
-
|
|
12909
|
-
|
|
12910
|
-
|
|
12911
|
-
|
|
12912
|
-
|
|
12913
|
-
|
|
12914
|
-
|
|
12915
|
-
}
|
|
12916
|
-
if (
|
|
12917
|
-
|
|
13940
|
+
if (legacy) {
|
|
13941
|
+
const receipts = raw.uncertainReceipts ?? {};
|
|
13942
|
+
if (!isRecord(receipts) || Object.keys(receipts).length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return null;
|
|
13943
|
+
for (const [id, receipt] of Object.entries(receipts)) {
|
|
13944
|
+
const normalized = normalizeAcquisitionReceipt(receipt, id);
|
|
13945
|
+
if (!normalized) return null;
|
|
13946
|
+
const key = receipt.resource === "core" ? "coreUnits" : "graphqlUnits";
|
|
13947
|
+
const next = state.legacyUnverified[key] + receipt.units;
|
|
13948
|
+
if (!Number.isSafeInteger(next)) return null;
|
|
13949
|
+
state.legacyUnverified[key] = next;
|
|
13950
|
+
state.legacyUnverified.receiptCount += 1;
|
|
13951
|
+
}
|
|
13952
|
+
if (raw.uncertainReceipts === undefined) {
|
|
13953
|
+
state.legacyUnverified.coreUnits = state.metrics.uncertainCoreUnits;
|
|
13954
|
+
state.legacyUnverified.graphqlUnits = state.metrics.uncertainGraphqlUnits;
|
|
13955
|
+
} else if (state.metrics.uncertainCoreUnits !== state.legacyUnverified.coreUnits ||
|
|
13956
|
+
state.metrics.uncertainGraphqlUnits !== state.legacyUnverified.graphqlUnits) return null;
|
|
13957
|
+
state.metrics.uncertainCoreUnits = 0;
|
|
13958
|
+
state.metrics.uncertainGraphqlUnits = 0;
|
|
13959
|
+
} else {
|
|
13960
|
+
if (!isRecord(raw.quotaProjections) || !isRecord(raw.legacyUnverified) ||
|
|
13961
|
+
Object.keys(raw.quotaProjections).length > ACQUISITION_MAX_QUOTA_PROJECTIONS ||
|
|
13962
|
+
!exactKeys(raw.legacyUnverified, ["coreUnits", "graphqlUnits", "receiptCount"]) ||
|
|
13963
|
+
Object.values(raw.legacyUnverified).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
|
|
13964
|
+
state.legacyUnverified = { ...raw.legacyUnverified };
|
|
13965
|
+
for (const [scopeHash, projection] of Object.entries(raw.quotaProjections)) {
|
|
13966
|
+
if (!/^[0-9a-f]{64}$/.test(scopeHash) || !exactKeys(projection, ["revision", "units", "at", "status", "accessKeys"]) ||
|
|
13967
|
+
!Number.isSafeInteger(projection.revision) || projection.revision < 0 ||
|
|
13968
|
+
!exactKeys(projection.units, RATE_RESOURCES) ||
|
|
13969
|
+
Object.values(projection.units).some((value) => !Number.isSafeInteger(value) || value < 0) ||
|
|
13970
|
+
!Number.isFinite(projection.at) || projection.at < 0 ||
|
|
13971
|
+
!["current", "unavailable"].includes(projection.status) ||
|
|
13972
|
+
!Array.isArray(projection.accessKeys) || projection.accessKeys.length > ACQUISITION_MAX_SUBSCRIPTIONS ||
|
|
13973
|
+
new Set(projection.accessKeys).size !== projection.accessKeys.length ||
|
|
13974
|
+
projection.accessKeys.some((key) => !/^[0-9a-f]{64}$/.test(key))) return null;
|
|
13975
|
+
state.quotaProjections[scopeHash] = { ...projection, units: { ...projection.units },
|
|
13976
|
+
accessKeys: [...projection.accessKeys] };
|
|
13977
|
+
}
|
|
12918
13978
|
}
|
|
12919
13979
|
for (const [key, value] of Object.entries(raw.aliases)) {
|
|
12920
13980
|
if (typeof key !== "string" || typeof value !== "string") return null;
|
|
@@ -12933,11 +13993,15 @@ function normalizeAcquisitionStore(raw) {
|
|
|
12933
13993
|
candidate.generation !== value.generation + 1 || !Number.isFinite(candidate.claimedAt) ||
|
|
12934
13994
|
!Number.isFinite(candidate.leaseUntil) || candidate.leaseUntil < candidate.claimedAt ||
|
|
12935
13995
|
typeof candidate.started !== "boolean") return null;
|
|
12936
|
-
|
|
12937
|
-
|
|
12938
|
-
|
|
12939
|
-
|
|
12940
|
-
|
|
13996
|
+
if (legacy) {
|
|
13997
|
+
const receipts = raw.uncertainReceipts ?? {};
|
|
13998
|
+
const receiptIds = candidate.receiptIds ?? [];
|
|
13999
|
+
if (!Array.isArray(receiptIds) || receiptIds.some((id) => typeof id !== "string" || !receipts[id])) return null;
|
|
14000
|
+
claim = null; // v1 producer authority is fenced by the new epoch.
|
|
14001
|
+
} else {
|
|
14002
|
+
if (Object.hasOwn(candidate, "receiptIds")) return null;
|
|
14003
|
+
claim = { ...candidate };
|
|
14004
|
+
}
|
|
12941
14005
|
}
|
|
12942
14006
|
const snapshot = value.snapshot === null ? null
|
|
12943
14007
|
: value.snapshot?.artifact
|
|
@@ -12990,37 +14054,63 @@ function loadAcquisitionMetadata(path) {
|
|
|
12990
14054
|
}
|
|
12991
14055
|
}
|
|
12992
14056
|
|
|
12993
|
-
function readAcquisitionSnapshot(path, record) {
|
|
12994
|
-
|
|
12995
|
-
|
|
12996
|
-
|
|
12997
|
-
|
|
12998
|
-
|
|
12999
|
-
|
|
13000
|
-
return { ok:
|
|
14057
|
+
function readAcquisitionSnapshot(path, record, { deadline = performance.now() + ACQUISITION_READ_MAX_MS } = {}) {
|
|
14058
|
+
let current = record;
|
|
14059
|
+
for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
|
|
14060
|
+
performance.now() < deadline; attempt += 1) {
|
|
14061
|
+
if (!current?.snapshot) return current === record ? { ok: true, value: null }
|
|
14062
|
+
: { ok: false, reason: "stale" };
|
|
14063
|
+
if (Array.isArray(current.snapshot.rows)) {
|
|
14064
|
+
return { ok: true, value: current.snapshot, replaced: current !== record };
|
|
14065
|
+
}
|
|
14066
|
+
try {
|
|
14067
|
+
const raw = readFileSync(join(acquisitionSnapshotRoot(path), current.snapshot.artifact), "utf8");
|
|
14068
|
+
if (Buffer.byteLength(raw) !== current.snapshot.bytes ||
|
|
14069
|
+
createHash("sha256").update(raw).digest("hex") !== current.snapshot.digest) {
|
|
14070
|
+
return { ok: false, reason: "corrupt" };
|
|
14071
|
+
}
|
|
14072
|
+
const parsed = JSON.parse(raw);
|
|
14073
|
+
const snapshot = normalizeAcquisitionSnapshot(
|
|
14074
|
+
parsed, current.query, current.generation, parsed.producerEpoch,
|
|
14075
|
+
);
|
|
14076
|
+
return snapshot ? { ok: true, value: snapshot, replaced: current !== record }
|
|
14077
|
+
: { ok: false, reason: "corrupt" };
|
|
14078
|
+
} catch (error) {
|
|
14079
|
+
if (error?.code !== "ENOENT") return { ok: false, reason: "corrupt", error };
|
|
14080
|
+
const latest = loadAcquisitionMetadata(path);
|
|
14081
|
+
if (!latest.ok) return latest;
|
|
14082
|
+
const replacement = latest.value.queries[current.query.queryKey];
|
|
14083
|
+
if (!replacement) return { ok: false, reason: "stale" };
|
|
14084
|
+
if (replacement.snapshot?.artifact === current.snapshot.artifact) {
|
|
14085
|
+
return { ok: false, reason: "corrupt", error };
|
|
14086
|
+
}
|
|
14087
|
+
current = replacement;
|
|
13001
14088
|
}
|
|
13002
|
-
const parsed = JSON.parse(raw);
|
|
13003
|
-
const snapshot = normalizeAcquisitionSnapshot(
|
|
13004
|
-
parsed, record.query, record.generation, parsed.producerEpoch,
|
|
13005
|
-
);
|
|
13006
|
-
return snapshot ? { ok: true, value: snapshot } : { ok: false, reason: "corrupt" };
|
|
13007
|
-
} catch (error) {
|
|
13008
|
-
return { ok: false, reason: "corrupt", error };
|
|
13009
14089
|
}
|
|
14090
|
+
return { ok: false, reason: "busy" };
|
|
13010
14091
|
}
|
|
13011
14092
|
|
|
13012
|
-
function hydrateAcquisitionStore(path, state) {
|
|
14093
|
+
function hydrateAcquisitionStore(path, state, deadline) {
|
|
13013
14094
|
for (const record of Object.values(state.queries)) {
|
|
13014
|
-
const loaded = readAcquisitionSnapshot(path, record);
|
|
14095
|
+
const loaded = readAcquisitionSnapshot(path, record, { deadline });
|
|
13015
14096
|
if (!loaded.ok) return loaded;
|
|
14097
|
+
if (loaded.replaced) return { ok: false, reason: "stale" };
|
|
13016
14098
|
record.snapshot = loaded.value;
|
|
13017
14099
|
}
|
|
13018
14100
|
return { ok: true, value: state };
|
|
13019
14101
|
}
|
|
13020
14102
|
|
|
13021
|
-
function loadAcquisitionStore(path) {
|
|
13022
|
-
const
|
|
13023
|
-
|
|
14103
|
+
function loadAcquisitionStore(path, { onMetadataLoaded = null } = {}) {
|
|
14104
|
+
const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
|
|
14105
|
+
for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
|
|
14106
|
+
performance.now() < deadline; attempt += 1) {
|
|
14107
|
+
const loaded = loadAcquisitionMetadata(path);
|
|
14108
|
+
if (!loaded.ok) return loaded;
|
|
14109
|
+
onMetadataLoaded?.();
|
|
14110
|
+
const hydrated = hydrateAcquisitionStore(path, loaded.value, deadline);
|
|
14111
|
+
if (hydrated.reason !== "stale") return hydrated;
|
|
14112
|
+
}
|
|
14113
|
+
return { ok: false, reason: "busy" };
|
|
13024
14114
|
}
|
|
13025
14115
|
|
|
13026
14116
|
function writeAcquisitionArtifact(root, artifact, payload) {
|
|
@@ -13193,8 +14283,8 @@ function trimAcquisitionStore(state, protectedQueryKey = null) {
|
|
|
13193
14283
|
|
|
13194
14284
|
async function runStartedAcquisitionTransport(markStarted, transport) {
|
|
13195
14285
|
const started = await markStarted();
|
|
13196
|
-
if (!started.ok) return started;
|
|
13197
|
-
if (started.value?.status !== "started") return { ok: false, reason: "already-started" };
|
|
14286
|
+
if (!started.ok) return { ...started, notStarted: true };
|
|
14287
|
+
if (started.value?.status !== "started") return { ok: false, reason: "already-started", notStarted: true };
|
|
13198
14288
|
return { ok: true, value: await transport() };
|
|
13199
14289
|
}
|
|
13200
14290
|
|
|
@@ -13244,7 +14334,7 @@ function createAcquisitionEngine({
|
|
|
13244
14334
|
}
|
|
13245
14335
|
const loaded = loadRecordSnapshot(record);
|
|
13246
14336
|
if (!loaded.ok) return loaded;
|
|
13247
|
-
item.generation =
|
|
14337
|
+
item.generation = loaded.value.generation;
|
|
13248
14338
|
try { item.callback?.(loaded.value); } catch { /* subscriber callback is isolated */ }
|
|
13249
14339
|
return { ok: true, value: true, snapshot: loaded.value };
|
|
13250
14340
|
}
|
|
@@ -13277,7 +14367,8 @@ function createAcquisitionEngine({
|
|
|
13277
14367
|
const record = loaded.value.queries[item.queryKey] ?? null;
|
|
13278
14368
|
if (!record?.snapshot) return { ok: true, value: record };
|
|
13279
14369
|
const snapshot = loadRecordSnapshot(record);
|
|
13280
|
-
return snapshot.ok ? { ok: true, value: { ...record,
|
|
14370
|
+
return snapshot.ok ? { ok: true, value: { ...record,
|
|
14371
|
+
generation: snapshot.value?.generation ?? record.generation, snapshot: snapshot.value } } : snapshot;
|
|
13281
14372
|
}
|
|
13282
14373
|
|
|
13283
14374
|
function heartbeat() {
|
|
@@ -13445,14 +14536,38 @@ function createAcquisitionEngine({
|
|
|
13445
14536
|
aliasConflict: matching.length > 1 } };
|
|
13446
14537
|
});
|
|
13447
14538
|
if (!result.ok) return result;
|
|
13448
|
-
|
|
13449
|
-
|
|
13450
|
-
|
|
13451
|
-
|
|
13452
|
-
local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation:
|
|
14539
|
+
// A publisher can canonicalize the query and remap this just-committed
|
|
14540
|
+
// shared subscription before its snapshot is hydrated. Register locally
|
|
14541
|
+
// first so every failure still has a cleanup owner, then follow the
|
|
14542
|
+
// subscription ID in fresh metadata rather than the old query key.
|
|
14543
|
+
local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation: 0,
|
|
13453
14544
|
claimNonce: null, pending: null });
|
|
13454
14545
|
ensureTimer();
|
|
13455
|
-
|
|
14546
|
+
const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
|
|
14547
|
+
for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
|
|
14548
|
+
performance.now() < deadline; attempt += 1) {
|
|
14549
|
+
const latest = load();
|
|
14550
|
+
if (!latest.ok) { unsubscribe(id); return latest; }
|
|
14551
|
+
const shared = latest.value.subscriptions[id];
|
|
14552
|
+
if (!shared || shared.nonce !== nonce) { unsubscribe(id); return { ok: false, reason: "stale" }; }
|
|
14553
|
+
const item = local.get(id);
|
|
14554
|
+
item.queryKey = shared.queryKey;
|
|
14555
|
+
const record = latest.value.queries[shared.queryKey];
|
|
14556
|
+
if (!record) continue;
|
|
14557
|
+
const loadedSnapshot = loadRecordSnapshot(record);
|
|
14558
|
+
if (!loadedSnapshot.ok) {
|
|
14559
|
+
if (["stale", "busy"].includes(loadedSnapshot.reason)) continue;
|
|
14560
|
+
unsubscribe(id);
|
|
14561
|
+
return loadedSnapshot;
|
|
14562
|
+
}
|
|
14563
|
+
if (loadedSnapshot.replaced) continue;
|
|
14564
|
+
result.value.queryKey = shared.queryKey;
|
|
14565
|
+
result.value.snapshot = loadedSnapshot.value;
|
|
14566
|
+
item.generation = loadedSnapshot.value?.generation ?? record.generation;
|
|
14567
|
+
return result;
|
|
14568
|
+
}
|
|
14569
|
+
unsubscribe(id);
|
|
14570
|
+
return { ok: false, reason: "busy" };
|
|
13456
14571
|
}
|
|
13457
14572
|
|
|
13458
14573
|
function updateDemand(id, demandInput) {
|
|
@@ -13509,11 +14624,6 @@ function createAcquisitionEngine({
|
|
|
13509
14624
|
const requestMetrics = produced?.requestMetrics === undefined
|
|
13510
14625
|
? {}
|
|
13511
14626
|
: normalizeAcquisitionMetrics(produced.requestMetrics, { partial: true });
|
|
13512
|
-
const receiptInputs = produced?.uncertainReceipts ?? [];
|
|
13513
|
-
const receipts = Array.isArray(receiptInputs)
|
|
13514
|
-
? receiptInputs.map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
|
|
13515
|
-
const receiptsValid = receipts && !receipts.includes(null) &&
|
|
13516
|
-
receipts.every((receipt) => receipt.accessKey === record.query.accessKey);
|
|
13517
14627
|
const loadedPrevious = loadRecordSnapshot(record);
|
|
13518
14628
|
if (!loadedPrevious.ok) return loadedPrevious;
|
|
13519
14629
|
if (loadedPrevious.value) record.snapshot = loadedPrevious.value;
|
|
@@ -13527,20 +14637,15 @@ function createAcquisitionEngine({
|
|
|
13527
14637
|
snapshot = { ...snapshot, lastChangedAt: loadedPrevious.value.lastChangedAt };
|
|
13528
14638
|
}
|
|
13529
14639
|
if (!snapshot) {
|
|
13530
|
-
settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics,
|
|
14640
|
+
settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics, [], now());
|
|
13531
14641
|
return { value: { ok: false, reason: "invalid" } };
|
|
13532
14642
|
}
|
|
13533
14643
|
const previous = record.snapshot;
|
|
13534
14644
|
const previousClaim = record.claim;
|
|
13535
|
-
if (requestMetrics === null
|
|
14645
|
+
if (requestMetrics === null) {
|
|
13536
14646
|
settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics ?? {}, [], now());
|
|
13537
14647
|
return { value: { ok: false, reason: "invalid" } };
|
|
13538
14648
|
}
|
|
13539
|
-
const receiptIds = new Set(Object.keys(state.uncertainReceipts ?? {}));
|
|
13540
|
-
const novelReceipts = receipts.filter((receipt) => !receiptIds.has(receipt.id));
|
|
13541
|
-
if (receiptIds.size + novelReceipts.length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) {
|
|
13542
|
-
return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
|
|
13543
|
-
}
|
|
13544
14649
|
record.generation = effectiveClaim.generation;
|
|
13545
14650
|
record.snapshot = snapshot;
|
|
13546
14651
|
record.claim = null;
|
|
@@ -13552,14 +14657,7 @@ function createAcquisitionEngine({
|
|
|
13552
14657
|
record.claim = previousClaim;
|
|
13553
14658
|
return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
|
|
13554
14659
|
}
|
|
13555
|
-
if (Object.hasOwn(requestMetrics, "uncertainCoreUnits") ||
|
|
13556
|
-
Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
|
|
13557
|
-
releaseKnownAcquisitionUncertainty(state, { ...record, claim: previousClaim }, requestMetrics);
|
|
13558
|
-
} else {
|
|
13559
|
-
releaseAcquisitionUncertainty(state, { ...record, claim: previousClaim });
|
|
13560
|
-
}
|
|
13561
14660
|
addAcquisitionRequestMetrics(state, requestMetrics);
|
|
13562
|
-
for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
|
|
13563
14661
|
settleAcquisitionWaiters(state, record.query.queryKey, effectiveClaim.generation, now());
|
|
13564
14662
|
return { value: { ok: true, value: { role: "producer", snapshot,
|
|
13565
14663
|
queryKey: record.query.queryKey, remapped } } };
|
|
@@ -13613,10 +14711,8 @@ function createAcquisitionEngine({
|
|
|
13613
14711
|
function settleFailure(id, claim, failure) {
|
|
13614
14712
|
const item = local.get(id);
|
|
13615
14713
|
const requestMetrics = normalizeAcquisitionMetrics(failure?.requestMetrics ?? {}, { partial: true });
|
|
13616
|
-
const receipts = Array.isArray(failure?.uncertainReceipts ?? [])
|
|
13617
|
-
? (failure?.uncertainReceipts ?? []).map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
|
|
13618
14714
|
const hold = failure?.hold ?? "primary";
|
|
13619
|
-
if (!item || !isRecord(claim) || requestMetrics === null ||
|
|
14715
|
+
if (!item || !isRecord(claim) || requestMetrics === null ||
|
|
13620
14716
|
!ACQUISITION_HOLD_REASONS.has(hold)) {
|
|
13621
14717
|
return { ok: false, reason: "invalid" };
|
|
13622
14718
|
}
|
|
@@ -13625,13 +14721,7 @@ function createAcquisitionEngine({
|
|
|
13625
14721
|
if (!record || record.claim?.nonce !== claim.nonce || record.claim.generation !== claim.generation) {
|
|
13626
14722
|
return { value: { ok: false, reason: "stale", definitive: true } };
|
|
13627
14723
|
}
|
|
13628
|
-
if (receipts.some((receipt) => receipt.accessKey !== record.query.accessKey) ||
|
|
13629
|
-
Object.keys(state.uncertainReceipts ?? {}).length +
|
|
13630
|
-
receipts.filter((receipt) => !state.uncertainReceipts?.[receipt.id]).length >
|
|
13631
|
-
ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return { ok: false, reason: "capacity" };
|
|
13632
14724
|
addAcquisitionRequestMetrics(state, requestMetrics);
|
|
13633
|
-
releaseKnownAcquisitionUncertainty(state, record, requestMetrics);
|
|
13634
|
-
for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
|
|
13635
14725
|
settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, now());
|
|
13636
14726
|
record.claim = null;
|
|
13637
14727
|
record.hold = { reason: hold, at: now() };
|
|
@@ -13654,7 +14744,11 @@ function createAcquisitionEngine({
|
|
|
13654
14744
|
const item = local.get(id);
|
|
13655
14745
|
if (!item || reason !== null && !ACQUISITION_HOLD_REASONS.has(reason) ||
|
|
13656
14746
|
expected !== null && (!isRecord(expected) || !TAB_KEYS.includes(expected.resource) ||
|
|
13657
|
-
typeof expected.accessKey !== "string" || expected.accessKey.length === 0
|
|
14747
|
+
typeof expected.accessKey !== "string" || expected.accessKey.length === 0 ||
|
|
14748
|
+
expected.currentHold !== undefined &&
|
|
14749
|
+
(!isRecord(expected.currentHold) ||
|
|
14750
|
+
!ACQUISITION_HOLD_REASONS.has(expected.currentHold.reason) ||
|
|
14751
|
+
!Number.isFinite(expected.currentHold.at)))) {
|
|
13658
14752
|
return { ok: false, reason: "invalid" };
|
|
13659
14753
|
}
|
|
13660
14754
|
return transact((state) => {
|
|
@@ -13662,6 +14756,11 @@ function createAcquisitionEngine({
|
|
|
13662
14756
|
if (!record) return { ok: false, reason: "stale" };
|
|
13663
14757
|
if (expected && (record.query.resource !== expected.resource ||
|
|
13664
14758
|
record.query.accessKey !== expected.accessKey)) return { ok: false, reason: "stale-access" };
|
|
14759
|
+
if (expected?.currentHold !== undefined &&
|
|
14760
|
+
(record.hold?.reason !== expected.currentHold.reason ||
|
|
14761
|
+
record.hold?.at !== expected.currentHold.at)) {
|
|
14762
|
+
return { ok: false, reason: "stale-hold" };
|
|
14763
|
+
}
|
|
13665
14764
|
if ((record.hold?.reason ?? null) === reason) return { changed: false, value: true };
|
|
13666
14765
|
const next = reason === null ? null : { reason, at: now() };
|
|
13667
14766
|
record.hold = next;
|
|
@@ -13693,28 +14792,10 @@ function createAcquisitionEngine({
|
|
|
13693
14792
|
if (now() >= acquisitionClaimDeadline(record.claim)) {
|
|
13694
14793
|
return { ok: false, reason: "stale" };
|
|
13695
14794
|
}
|
|
13696
|
-
const receiptIds = [];
|
|
13697
|
-
for (const resource of RATE_RESOURCES) {
|
|
13698
|
-
if (costs[resource] <= 0) continue;
|
|
13699
|
-
const epoch = typeof supplied?.epochs?.[resource] === "string" && supplied.epochs[resource].length > 0
|
|
13700
|
-
? supplied.epochs[resource] : "unknown";
|
|
13701
|
-
const receipt = {
|
|
13702
|
-
id: `${reservationId}:${resource}`,
|
|
13703
|
-
reservationId,
|
|
13704
|
-
accessKey: record.query.accessKey,
|
|
13705
|
-
resource,
|
|
13706
|
-
epoch,
|
|
13707
|
-
units: costs[resource],
|
|
13708
|
-
startedAt: now(),
|
|
13709
|
-
};
|
|
13710
|
-
if (!addAcquisitionReceipt(state, receipt)) return { ok: false, reason: "capacity" };
|
|
13711
|
-
receiptIds.push(receipt.id);
|
|
13712
|
-
}
|
|
13713
14795
|
record.claim.started = true;
|
|
13714
14796
|
record.claim.startedAt = now();
|
|
13715
14797
|
record.claim.reservationId = reservationId;
|
|
13716
14798
|
record.claim.startReceipt = startReceipt;
|
|
13717
|
-
record.claim.receiptIds = receiptIds;
|
|
13718
14799
|
return { value: { status: "started", reservationId } };
|
|
13719
14800
|
});
|
|
13720
14801
|
}
|
|
@@ -13764,7 +14845,8 @@ function createAcquisitionEngine({
|
|
|
13764
14845
|
const claimed = transact((state) => {
|
|
13765
14846
|
const reaped = reapAcquisitionSubscriptions(state, now(), kill);
|
|
13766
14847
|
const subscription = state.subscriptions[id];
|
|
13767
|
-
const
|
|
14848
|
+
const queryKey = subscription?.queryKey;
|
|
14849
|
+
const record = state.queries[queryKey];
|
|
13768
14850
|
if (!subscription || subscription.nonce !== item.nonce || !record) return { ok: false, reason: "stale" };
|
|
13769
14851
|
const requestedGeneration = force
|
|
13770
14852
|
? Math.max(subscription.requestedGeneration,
|
|
@@ -13776,8 +14858,8 @@ function createAcquisitionEngine({
|
|
|
13776
14858
|
}
|
|
13777
14859
|
if (requestedGeneration <= record.generation && record.snapshot?.nextDueAt > now()) {
|
|
13778
14860
|
return { changed: reaped > 0 || demandChanged,
|
|
13779
|
-
value: { role: "follower", reason: "fresh", snapshot: record.snapshot,
|
|
13780
|
-
sharingCount: acquisitionSharingCount(state,
|
|
14861
|
+
value: { role: "follower", reason: "fresh", queryKey, snapshot: record.snapshot,
|
|
14862
|
+
sharingCount: acquisitionSharingCount(state, queryKey) } };
|
|
13781
14863
|
}
|
|
13782
14864
|
if (record.claim) {
|
|
13783
14865
|
const absoluteDeadline = acquisitionClaimDeadline(record.claim);
|
|
@@ -13792,10 +14874,10 @@ function createAcquisitionEngine({
|
|
|
13792
14874
|
}
|
|
13793
14875
|
return { changed: reaped > 0 || demandChanged || joined,
|
|
13794
14876
|
value: { role: "follower", reason: expired ? `owner-${owner}` : "claimed",
|
|
13795
|
-
snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state,
|
|
14877
|
+
queryKey, snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state, queryKey) } };
|
|
13796
14878
|
}
|
|
13797
14879
|
}
|
|
13798
|
-
const nonce = createId("acquisition-claim",
|
|
14880
|
+
const nonce = createId("acquisition-claim", queryKey, record.generation + 1);
|
|
13799
14881
|
record.claim = {
|
|
13800
14882
|
pid,
|
|
13801
14883
|
nonce,
|
|
@@ -13809,11 +14891,13 @@ function createAcquisitionEngine({
|
|
|
13809
14891
|
subscription.waitingSinceAt = null;
|
|
13810
14892
|
record.lastUsedAt = now();
|
|
13811
14893
|
return { value: { role: "producer", nonce, generation: record.claim.generation,
|
|
13812
|
-
query: record.query, snapshot: record.snapshot,
|
|
13813
|
-
|
|
14894
|
+
queryKey, query: record.query, snapshot: record.snapshot,
|
|
14895
|
+
demand: aggregateAcquisitionDemand(state, queryKey),
|
|
14896
|
+
sharingCount: acquisitionSharingCount(state, queryKey) } };
|
|
13814
14897
|
});
|
|
14898
|
+
if (claimed.ok && claimed.value?.queryKey) item.queryKey = claimed.value.queryKey;
|
|
13815
14899
|
if (claimed.ok && claimed.value?.snapshot) {
|
|
13816
|
-
const record = claimed.state.queries[
|
|
14900
|
+
const record = claimed.state.queries[claimed.value.queryKey];
|
|
13817
14901
|
const loadedSnapshot = loadRecordSnapshot(record);
|
|
13818
14902
|
if (!loadedSnapshot.ok) return loadedSnapshot;
|
|
13819
14903
|
claimed.value.snapshot = loadedSnapshot.value;
|
|
@@ -13921,21 +15005,58 @@ function createAcquisitionEngine({
|
|
|
13921
15005
|
});
|
|
13922
15006
|
}
|
|
13923
15007
|
|
|
13924
|
-
function
|
|
13925
|
-
if (
|
|
15008
|
+
function syncQuotaProjection(scope) {
|
|
15009
|
+
if (!isRecord(scope) || !/^[0-9a-f]{64}$/.test(scope.hash ?? "")) {
|
|
13926
15010
|
return { ok: false, reason: "invalid" };
|
|
13927
15011
|
}
|
|
15012
|
+
// Quota is read before the acquisition lock. This diagnostic copy never
|
|
15013
|
+
// grants requests and a failed read cannot block source publication.
|
|
15014
|
+
const observed = inspectGovernor(scope, now());
|
|
15015
|
+
if (!observed.ok) {
|
|
15016
|
+
return transact((state) => {
|
|
15017
|
+
const previous = state.quotaProjections[scope.hash];
|
|
15018
|
+
if (!previous || previous.status === "unavailable") return { changed: false, value: false };
|
|
15019
|
+
previous.status = "unavailable";
|
|
15020
|
+
return { value: false };
|
|
15021
|
+
});
|
|
15022
|
+
}
|
|
15023
|
+
const ledger = observed.value;
|
|
15024
|
+
const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
15025
|
+
Object.values(ledger.reservations).reduce((sum, reservation) =>
|
|
15026
|
+
sum + reservationCost(reservation, resource, ledger.leases, now()),
|
|
15027
|
+
ledger.debt[resource].unresolvedUnits + ledger.debt[resource].quiescentUnits +
|
|
15028
|
+
reservationCost(ledger.controlReceipts[resource], resource, ledger.leases, now()))]));
|
|
15029
|
+
if (Object.values(units).some((value) => !Number.isSafeInteger(value) || value < 0)) {
|
|
15030
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
15031
|
+
}
|
|
13928
15032
|
return transact((state) => {
|
|
13929
|
-
|
|
13930
|
-
|
|
13931
|
-
|
|
13932
|
-
|
|
13933
|
-
|
|
13934
|
-
|
|
13935
|
-
|
|
13936
|
-
|
|
15033
|
+
const activeKeys = new Set(Object.values(state.subscriptions)
|
|
15034
|
+
.map((subscription) => state.queries[subscription.queryKey]?.query.accessKey)
|
|
15035
|
+
.filter(Boolean));
|
|
15036
|
+
const currentAccessKey = /^[0-9a-f]{64}$/.test(scope.accessKey ?? "")
|
|
15037
|
+
? scope.accessKey : null;
|
|
15038
|
+
if (currentAccessKey) {
|
|
15039
|
+
for (const [hash, projection] of Object.entries(state.quotaProjections)) {
|
|
15040
|
+
if (hash !== scope.hash) projection.accessKeys = projection.accessKeys.filter((key) =>
|
|
15041
|
+
key !== currentAccessKey);
|
|
15042
|
+
}
|
|
13937
15043
|
}
|
|
13938
|
-
|
|
15044
|
+
const previous = state.quotaProjections[scope.hash];
|
|
15045
|
+
if (previous && previous.revision > ledger.revision) return { changed: false, value: false };
|
|
15046
|
+
if (!previous && Object.keys(state.quotaProjections).length >= ACQUISITION_MAX_QUOTA_PROJECTIONS) {
|
|
15047
|
+
const oldest = Object.entries(state.quotaProjections)
|
|
15048
|
+
.filter(([, projection]) => !projection.accessKeys.some((key) => activeKeys.has(key)))
|
|
15049
|
+
.sort((a, b) => a[1].at - b[1].at)[0];
|
|
15050
|
+
if (!oldest) return { changed: false, value: false };
|
|
15051
|
+
delete state.quotaProjections[oldest[0]];
|
|
15052
|
+
}
|
|
15053
|
+
const accessKeys = previous?.accessKeys.filter((key) => activeKeys.has(key)) ?? [];
|
|
15054
|
+
if (currentAccessKey && !accessKeys.includes(currentAccessKey) &&
|
|
15055
|
+
accessKeys.length < ACQUISITION_MAX_SUBSCRIPTIONS) accessKeys.push(currentAccessKey);
|
|
15056
|
+
state.quotaProjections[scope.hash] = {
|
|
15057
|
+
revision: ledger.revision, units, at: now(), status: "current", accessKeys,
|
|
15058
|
+
};
|
|
15059
|
+
return { value: true };
|
|
13939
15060
|
});
|
|
13940
15061
|
}
|
|
13941
15062
|
|
|
@@ -13948,7 +15069,7 @@ function createAcquisitionEngine({
|
|
|
13948
15069
|
inspect,
|
|
13949
15070
|
diagnostics,
|
|
13950
15071
|
recordMetrics,
|
|
13951
|
-
|
|
15072
|
+
syncQuotaProjection,
|
|
13952
15073
|
setHold,
|
|
13953
15074
|
close,
|
|
13954
15075
|
path,
|
|
@@ -13964,6 +15085,27 @@ function setRuntimeAcquisitionHold(engine, subscriptions, resource, reason, acce
|
|
|
13964
15085
|
return engine.setHold(subscription.id, reason, { resource, accessKey });
|
|
13965
15086
|
}
|
|
13966
15087
|
|
|
15088
|
+
function clearScheduledRuntimeControlHold(engine, subscriptions, resource, accessKey, observedAt) {
|
|
15089
|
+
const subscription = subscriptions?.get(resource);
|
|
15090
|
+
if (!subscription?.id) return { ok: false, reason: "stale" };
|
|
15091
|
+
if (typeof accessKey !== "string" || !Number.isFinite(observedAt)) {
|
|
15092
|
+
return { ok: false, reason: "stale-access" };
|
|
15093
|
+
}
|
|
15094
|
+
const inspected = engine.inspect(subscription.id);
|
|
15095
|
+
if (!inspected.ok) return inspected;
|
|
15096
|
+
const record = inspected.value;
|
|
15097
|
+
if (record?.query?.resource !== resource || record.query.accessKey !== accessKey) {
|
|
15098
|
+
return { ok: false, reason: "stale-access" };
|
|
15099
|
+
}
|
|
15100
|
+
const hold = record.hold;
|
|
15101
|
+
if (!hold) return { ok: true, value: "already-clear" };
|
|
15102
|
+
if (hold.reason !== "coordination") return { ok: true, value: "retained" };
|
|
15103
|
+
if (hold.at >= observedAt) return { ok: false, reason: "newer-hold" };
|
|
15104
|
+
const cleared = engine.setHold(subscription.id, null,
|
|
15105
|
+
{ resource, accessKey, currentHold: hold });
|
|
15106
|
+
return cleared.ok ? { ok: true, value: "cleared" } : cleared;
|
|
15107
|
+
}
|
|
15108
|
+
|
|
13967
15109
|
// The one place the cadence table is decided. It used to be spread across a
|
|
13968
15110
|
// fixed active floor, a fixed four-floor background slot, and a Security-only
|
|
13969
15111
|
// unchanged rule -- three policies that could disagree, and did: Security
|
|
@@ -13977,7 +15119,10 @@ function pollPolicyInterval({
|
|
|
13977
15119
|
unchangedCount = 0,
|
|
13978
15120
|
inProgressCI = false,
|
|
13979
15121
|
background = "all",
|
|
15122
|
+
requested = false,
|
|
13980
15123
|
}) {
|
|
15124
|
+
if (background === "on-demand" &&
|
|
15125
|
+
(demand !== "active" || (tab !== "actions" && !requested))) return Number.POSITIVE_INFINITY;
|
|
13981
15126
|
if (demand === "none") return Number.POSITIVE_INFINITY;
|
|
13982
15127
|
if (demand === "inactive") {
|
|
13983
15128
|
if (background === "off") return Number.POSITIVE_INFINITY;
|
|
@@ -14105,6 +15250,7 @@ function pollSchedule({
|
|
|
14105
15250
|
demand: isActive ? "active" : "inactive",
|
|
14106
15251
|
unchangedCount: states[key]?.unchangedCount ?? 0,
|
|
14107
15252
|
inProgressCI: states[key]?.inProgressCI === true,
|
|
15253
|
+
requested: states[key]?.requested === true,
|
|
14108
15254
|
background,
|
|
14109
15255
|
});
|
|
14110
15256
|
if (!Number.isFinite(step)) {
|
|
@@ -14249,7 +15395,7 @@ function governorDataReady(refreshResult, snapshot, activeKey, nowMs) {
|
|
|
14249
15395
|
return RATE_RESOURCES.every((resource) => {
|
|
14250
15396
|
if (costs[resource] <= 0) return true;
|
|
14251
15397
|
const budget = snapshot.value.budgets[resource];
|
|
14252
|
-
const available =
|
|
15398
|
+
const available = governorAvailableForGrant(snapshot.value, resource, nowMs);
|
|
14253
15399
|
return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource) &&
|
|
14254
15400
|
budget.blockUntil <= nowMs && available.mode === "open" &&
|
|
14255
15401
|
available.spendable >= costs[resource];
|
|
@@ -14456,6 +15602,333 @@ function coordinationNotice(reason) {
|
|
|
14456
15602
|
return "Can't coordinate API use — retrying";
|
|
14457
15603
|
}
|
|
14458
15604
|
|
|
15605
|
+
// Recovery diagnostics have no request payloads, credential identifiers or
|
|
15606
|
+
// owner capabilities. The journal is advisory: a failed write must never deny
|
|
15607
|
+
// an otherwise valid data operation. The quota ledger remains authoritative.
|
|
15608
|
+
const RECOVERY_JOURNAL_MAX_ENTRIES = 128;
|
|
15609
|
+
const RECOVERY_JOURNAL_MAX_BYTES = 128 * 1024;
|
|
15610
|
+
const RECOVERY_ENTRY_MAX_BYTES = 1024;
|
|
15611
|
+
const RECOVERY_COPY = Object.freeze({
|
|
15612
|
+
busy: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
|
|
15613
|
+
stale: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
|
|
15614
|
+
"identity-busy": ["GitHub budget check is shared", "Retry automatically after the current check", "Budget check; retry"],
|
|
15615
|
+
"transport-busy": ["Shared GitHub request is in progress", "Retry automatically", "Request; retry"],
|
|
15616
|
+
unwritable: ["Local coordination cannot be written", "Run gh-glance --doctor; check storage permissions", "Store blocked; doctor"],
|
|
15617
|
+
corrupt: ["Local coordination is corrupt", "Run gh-glance --doctor; preserve the reported state file", "State corrupt; doctor"],
|
|
15618
|
+
capacity: ["Local coordination is full", "Run gh-glance --doctor; preserve the reported state file", "State full; doctor"],
|
|
15619
|
+
"accounting-overflow": ["Local coordination accounting overflow", "Run gh-glance --doctor; preserve the reported state file", "Overflow; doctor"],
|
|
15620
|
+
"reservations-invalid": ["Local coordination reservation invalid", "Run gh-glance --doctor; preserve the reported state file", "Reservation; doctor"],
|
|
15621
|
+
"external-factor-invalid": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
|
|
15622
|
+
"pacing-invalid": ["Local coordination pacing invalid", "Run gh-glance --doctor; preserve the reported state file", "Pacing invalid; doctor"],
|
|
15623
|
+
"budget-resource": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
|
|
15624
|
+
"legacy-unresolved": ["Older requests still reserve quota", "Verify old children ended; a verified system restart can resolve unknown ownership", "Old debt; restart"],
|
|
15625
|
+
"legacy-corrupt": ["Older coordination evidence is corrupt", "Run gh-glance --doctor; preserve the reported state file", "Old state; doctor"],
|
|
15626
|
+
"migration-hold": ["Older session still uses coordination", "Wait for the older quota reset; retry automatically at the shown time", "Old quota; wait"],
|
|
15627
|
+
"restart-required": ["Older session still uses coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
|
|
15628
|
+
"identity-backoff": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
|
|
15629
|
+
"identity-capacity": ["GitHub budget check allowance is full", "Retry automatically when the rolling window renews", "Budget check; retry"],
|
|
15630
|
+
"budget-unknown": ["GitHub budget check pending", "Retry automatically", "Budget check; retry"],
|
|
15631
|
+
"budget-stale": ["GitHub budget evidence is old", "Retry the shared budget check automatically", "Budget old; retry"],
|
|
15632
|
+
"budget-future": ["Clock changed; budget needs recheck", "Retry the shared budget check automatically", "Clock; retry"],
|
|
15633
|
+
"probe-failed": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
|
|
15634
|
+
"block-unpublished": ["Saving GitHub request limit", "Retry sharing the hold automatically", "Saving hold; retry"],
|
|
15635
|
+
"budget-reset": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
|
|
15636
|
+
"rate-limit": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
|
|
15637
|
+
"rate-limited": ["GitHub request limit", "Retry automatically after the shared hold", "Limit; retry"],
|
|
15638
|
+
"local-reserve": ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
|
|
15639
|
+
reserve: ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
|
|
15640
|
+
blocked: ["GitHub request hold", "Retry automatically when the hold ends", "Request hold; retry"],
|
|
15641
|
+
secondary: ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
|
|
15642
|
+
"secondary-rate-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
|
|
15643
|
+
"abuse-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
|
|
15644
|
+
"throttle-paused": ["Repeated GitHub request limit", "Press r to retry after checking the hold", "Limit; press r"],
|
|
15645
|
+
"network-outage": ["GitHub connection failed", "Retry automatically when the connection returns", "Network; retry"],
|
|
15646
|
+
"credential-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
|
|
15647
|
+
"identity-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
|
|
15648
|
+
disconnected: ["Collector connection unavailable", "Restore the selected collector connection", "Collector; connection"],
|
|
15649
|
+
"app-auth": ["GitHub App authorization unavailable", "Restore the selected App authorization", "App; restore auth"],
|
|
15650
|
+
"security-incomplete": ["Security data incomplete", "Run gh-glance --doctor --probe for endpoint access", "Security; doctor"],
|
|
15651
|
+
"receipt-retry": ["Saving request result", "Retry automatically without repeating the request", "Saving; retry"],
|
|
15652
|
+
"interrupted-request": ["Recovering interrupted request", "Retry source observation when quota allows", "Interrupted; retry"],
|
|
15653
|
+
"request-queue": ["Recovering local request queue", "Retry automatically after compaction", "Queue; retry"],
|
|
15654
|
+
"acquisition-busy": ["Acquisition store is busy", "Check the shared lock; retry automatically", "Store lock; retry"],
|
|
15655
|
+
"clock-recovery": ["Rechecking after sleep or clock change", "Retry source observation automatically", "Clock; retry"],
|
|
15656
|
+
"auth-problem": ["GitHub authorization failed", "Run gh auth status, then restore authorization", "Auth; gh auth status"],
|
|
15657
|
+
unavailable: ["Repository or endpoint unavailable", "Check the selected repository and permissions", "Repo; check access"],
|
|
15658
|
+
"source-incomplete": ["GitHub response incomplete", "Retry source observation automatically", "Incomplete; retry"],
|
|
15659
|
+
"unknown-scope": ["GitHub account scope unavailable", "Run gh auth status; retry when identity is verified", "Login; gh auth status"],
|
|
15660
|
+
"version-mismatch": ["Older session cannot share coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
|
|
15661
|
+
"unsafe-permissions": ["Local coordination permissions are unsafe", "Run gh-glance --doctor; repair directory permissions", "Permissions; doctor"],
|
|
15662
|
+
"disk-full": ["Local coordination storage is full", "Run gh-glance --doctor; free local storage", "Disk full; doctor"],
|
|
15663
|
+
coordination: ["Local coordination unavailable", "Run gh-glance --doctor; retry when storage works", "Coordination; doctor"],
|
|
15664
|
+
});
|
|
15665
|
+
|
|
15666
|
+
function recoveryCause({ reason, resource = null, at = Date.now(), retryAt = null,
|
|
15667
|
+
sourceAt = null, receiptUnits = null, debtUnits = null, oldestUnresolvedAt = null,
|
|
15668
|
+
observerAt = null, terminalizationBacklog = null, lastFailedTransition = null,
|
|
15669
|
+
origin = "coordination", tab = null } = {}) {
|
|
15670
|
+
const code = typeof reason === "string" && Object.hasOwn(RECOVERY_COPY, reason)
|
|
15671
|
+
? reason : "coordination";
|
|
15672
|
+
const copy = RECOVERY_COPY[code] ?? RECOVERY_COPY.coordination;
|
|
15673
|
+
const time = (value) => Number.isFinite(value) && value >= 0 ? value : null;
|
|
15674
|
+
const units = (value) => Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
15675
|
+
return {
|
|
15676
|
+
code, resource: RATE_RESOURCES.includes(resource) ? resource : null,
|
|
15677
|
+
origin: origin === "source" ? "source" : "coordination",
|
|
15678
|
+
tab: TAB_KEYS.includes(tab) ? tab : null,
|
|
15679
|
+
at: time(at) ?? Date.now(), firstAt: time(at) ?? Date.now(),
|
|
15680
|
+
retryAt: time(retryAt), sourceAt: time(sourceAt),
|
|
15681
|
+
receiptUnits: units(receiptUnits), debtUnits: units(debtUnits),
|
|
15682
|
+
oldestUnresolvedAt: time(oldestUnresolvedAt), observerAt: time(observerAt),
|
|
15683
|
+
terminalizationBacklog: units(terminalizationBacklog),
|
|
15684
|
+
lastFailedTransition: typeof lastFailedTransition === "string" && Object.hasOwn(RECOVERY_COPY, lastFailedTransition)
|
|
15685
|
+
? lastFailedTransition : code,
|
|
15686
|
+
summary: copy[0], action: copy[1], short: copy[2],
|
|
15687
|
+
};
|
|
15688
|
+
}
|
|
15689
|
+
|
|
15690
|
+
function shouldClearRecoveryCause(cause, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
|
|
15691
|
+
if (!cause) return false;
|
|
15692
|
+
if (scheduledHold !== null) {
|
|
15693
|
+
return scheduledHold.ok === true && scheduledHold.value === "cleared" &&
|
|
15694
|
+
cause.origin === "coordination" && ["busy", "stale", "acquisition-busy"].includes(cause.code);
|
|
15695
|
+
}
|
|
15696
|
+
return cause.origin !== "source" || sourceSuccess && cause.tab === tab;
|
|
15697
|
+
}
|
|
15698
|
+
|
|
15699
|
+
function recoveryRetryAt(decision, budget, observer, nowMs) {
|
|
15700
|
+
if (budget?.blockUntil > nowMs) return budget.blockUntil;
|
|
15701
|
+
if (["reservations-invalid", "external-factor-invalid", "pacing-invalid", "budget-resource",
|
|
15702
|
+
"accounting-overflow", "corrupt", "unwritable", "capacity"].includes(decision?.reason)) return null;
|
|
15703
|
+
const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision?.reason);
|
|
15704
|
+
if (decision?.mode === "probe" && !resetHold) {
|
|
15705
|
+
return observer?.nextAt > nowMs ? observer.nextAt : null;
|
|
15706
|
+
}
|
|
15707
|
+
const deadline = decision?.retryAt ?? decision?.resetMs ?? budget?.resetMs;
|
|
15708
|
+
return deadline > nowMs ? deadline : null;
|
|
15709
|
+
}
|
|
15710
|
+
|
|
15711
|
+
function terminalizationRecoveryCode(terminal) {
|
|
15712
|
+
if (terminal?.status === "retryable") return "receipt-retry";
|
|
15713
|
+
if (terminal?.status === "compacted") return "interrupted-request";
|
|
15714
|
+
if (terminal?.reason === "completion-capacity") return "capacity";
|
|
15715
|
+
return terminal?.reason ?? terminal?.status ?? "coordination";
|
|
15716
|
+
}
|
|
15717
|
+
|
|
15718
|
+
function requestFailureRecoveryCode(error, verdict) {
|
|
15719
|
+
if (verdict === "rate-limited") return "rate-limited";
|
|
15720
|
+
if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) return error.coordinationReason;
|
|
15721
|
+
if (verdict === "other" && error?.httpStarted === true) return "interrupted-request";
|
|
15722
|
+
return verdict === "other" ? "network-outage" : verdict;
|
|
15723
|
+
}
|
|
15724
|
+
|
|
15725
|
+
function requestFailureRecovery(error, verdict) {
|
|
15726
|
+
const reason = requestFailureRecoveryCode(error, verdict);
|
|
15727
|
+
return { reason, origin: reason === "transport-busy" ? "coordination" : "source" };
|
|
15728
|
+
}
|
|
15729
|
+
|
|
15730
|
+
function wallClockRecovery(previousAt, at, intervalMs) {
|
|
15731
|
+
if (!Number.isFinite(previousAt) || !Number.isFinite(at)) return null;
|
|
15732
|
+
return at < previousAt || at - previousAt > Math.max(30_000, 3 * intervalMs)
|
|
15733
|
+
? "clock-recovery" : null;
|
|
15734
|
+
}
|
|
15735
|
+
|
|
15736
|
+
function collectorSourceAdvanced(previousAt, sourceAt) {
|
|
15737
|
+
return Number.isFinite(previousAt) && Number.isFinite(sourceAt) && sourceAt > previousAt;
|
|
15738
|
+
}
|
|
15739
|
+
|
|
15740
|
+
function cleanupQueueForResource(queue, resource) {
|
|
15741
|
+
let count = 0;
|
|
15742
|
+
for (const item of queue) if (tabRequestCost(item.cleanupKey)?.[resource] > 0) count += 1;
|
|
15743
|
+
return count;
|
|
15744
|
+
}
|
|
15745
|
+
|
|
15746
|
+
function presentRecovery(cause, { nowMs = Date.now(), cols = 80 } = {}) {
|
|
15747
|
+
if (!cause) return [];
|
|
15748
|
+
const width = Math.max(1, Number.isSafeInteger(cols) ? cols - 1 : 79);
|
|
15749
|
+
const resource = cause.resource === "graphql" ? "GraphQL" : cause.resource === "core" ? "Core" : "GitHub";
|
|
15750
|
+
const age = Number.isFinite(cause.sourceAt) && cause.sourceAt <= nowMs
|
|
15751
|
+
? `cached ${formatDuration(nowMs - cause.sourceAt)}` : "cached age unavailable";
|
|
15752
|
+
const retry = Number.isFinite(cause.retryAt)
|
|
15753
|
+
? `; next ${statusInterval(cause.retryAt, nowMs)}` : "";
|
|
15754
|
+
const summary = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
|
|
15755
|
+
? `Older requests still reserve ${cause.debtUnits} units` : cause.summary;
|
|
15756
|
+
if (width < 60) {
|
|
15757
|
+
const action = cause.action.toLowerCase();
|
|
15758
|
+
const narrowAction = cause.code === "legacy-unresolved" ? "Check children; restart"
|
|
15759
|
+
: action.includes("gh-glance --doctor") ? "Run gh-glance --doctor"
|
|
15760
|
+
: action.includes("gh auth") ? "Run gh auth status"
|
|
15761
|
+
: action.includes("selected repository") ? "Check repository access"
|
|
15762
|
+
: action.includes("connection") ? "Restore connection"
|
|
15763
|
+
: action.includes("app authorization") ? "Restore App auth"
|
|
15764
|
+
: action.includes("permissions") ? "Check permissions"
|
|
15765
|
+
: action.includes("restart") ? "Restart and recheck"
|
|
15766
|
+
: action.includes("press r") ? "Press r to retry"
|
|
15767
|
+
: "Retry automatically";
|
|
15768
|
+
const narrowDeadline = Number.isFinite(cause.retryAt)
|
|
15769
|
+
? statusInterval(cause.retryAt, nowMs) : null;
|
|
15770
|
+
const timedAction = narrowDeadline && cause.code === "migration-hold"
|
|
15771
|
+
? `Wait; next ${narrowDeadline}`
|
|
15772
|
+
: narrowDeadline && narrowAction === "Retry automatically"
|
|
15773
|
+
? `Retry; next ${narrowDeadline}`
|
|
15774
|
+
: narrowDeadline && narrowAction === "Press r to retry"
|
|
15775
|
+
? `Press r after ${narrowDeadline}` : narrowAction;
|
|
15776
|
+
const short = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
|
|
15777
|
+
? `Debt ${cause.debtUnits} units` : cause.short.split(";")[0];
|
|
15778
|
+
return [`${resource}: ${short}`.slice(0, width), timedAction.slice(0, width), age.slice(0, width)];
|
|
15779
|
+
}
|
|
15780
|
+
const full = `${resource}: ${summary}; ${cause.action}${retry}; ${age}`;
|
|
15781
|
+
if (full.length <= width) return [full];
|
|
15782
|
+
return [`${resource}: ${summary}`.slice(0, width),
|
|
15783
|
+
`${cause.action}${retry}`.slice(0, width), age.slice(0, width)];
|
|
15784
|
+
}
|
|
15785
|
+
|
|
15786
|
+
function recoveryJournalPath(scope) {
|
|
15787
|
+
return `${scope.path}.recovery.json`;
|
|
15788
|
+
}
|
|
15789
|
+
|
|
15790
|
+
function emptyRecoveryJournal() {
|
|
15791
|
+
return { version: 1, current: { core: null, graphql: null }, entries: [] };
|
|
15792
|
+
}
|
|
15793
|
+
|
|
15794
|
+
const RECOVERY_EVENT_KEYS = ["code", "resource", "at", "firstAt", "retryAt", "sourceAt",
|
|
15795
|
+
"receiptUnits", "debtUnits", "oldestUnresolvedAt", "observerAt", "terminalizationBacklog",
|
|
15796
|
+
"lastFailedTransition", "origin", "tab"];
|
|
15797
|
+
|
|
15798
|
+
function recoveryEvent(entry) {
|
|
15799
|
+
if (!isRecord(entry)) return null;
|
|
15800
|
+
if (exactKeys(entry, ["code", "resource", "at"]) && entry.code === "clear" &&
|
|
15801
|
+
RATE_RESOURCES.includes(entry.resource) && Number.isFinite(entry.at)) return entry;
|
|
15802
|
+
// An earlier clock sample can arrive after a later one for the same cause.
|
|
15803
|
+
// Repair only this bounded timestamp skew; every other field stays subject
|
|
15804
|
+
// to the strict journal validation below.
|
|
15805
|
+
const event = Number.isFinite(entry.at) && Number.isFinite(entry.firstAt) && entry.firstAt > entry.at
|
|
15806
|
+
? { ...entry, firstAt: entry.at } : entry;
|
|
15807
|
+
if (!exactKeys(event, RECOVERY_EVENT_KEYS) || !RATE_RESOURCES.includes(event.resource) ||
|
|
15808
|
+
typeof event.code !== "string" || !Object.hasOwn(RECOVERY_COPY, event.code) ||
|
|
15809
|
+
!Number.isFinite(event.at) || !Number.isFinite(event.firstAt) ||
|
|
15810
|
+
["retryAt", "sourceAt", "oldestUnresolvedAt", "observerAt"].some((key) =>
|
|
15811
|
+
event[key] !== null && (!Number.isFinite(event[key]) || event[key] < 0)) ||
|
|
15812
|
+
["receiptUnits", "debtUnits", "terminalizationBacklog"].some((key) =>
|
|
15813
|
+
event[key] !== null && (!Number.isSafeInteger(event[key]) || event[key] < 0)) ||
|
|
15814
|
+
typeof event.lastFailedTransition !== "string" ||
|
|
15815
|
+
!Object.hasOwn(RECOVERY_COPY, event.lastFailedTransition) ||
|
|
15816
|
+
!["source", "coordination"].includes(event.origin) ||
|
|
15817
|
+
(event.tab !== null && !TAB_KEYS.includes(event.tab)) ||
|
|
15818
|
+
Buffer.byteLength(JSON.stringify(event)) > RECOVERY_ENTRY_MAX_BYTES) return null;
|
|
15819
|
+
return event;
|
|
15820
|
+
}
|
|
15821
|
+
|
|
15822
|
+
function visibleRecoveryEvent(entry) {
|
|
15823
|
+
if (!entry || entry.code === "clear") return entry;
|
|
15824
|
+
const copy = RECOVERY_COPY[entry.code] ?? RECOVERY_COPY.coordination;
|
|
15825
|
+
return { ...entry, summary: copy[0], action: copy[1], short: copy[2] };
|
|
15826
|
+
}
|
|
15827
|
+
|
|
15828
|
+
function readRecoveryTransitions(scope) {
|
|
15829
|
+
const path = recoveryJournalPath(scope);
|
|
15830
|
+
let raw;
|
|
15831
|
+
try {
|
|
15832
|
+
const file = lstatSync(path);
|
|
15833
|
+
if (!file.isFile() || file.size > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
|
|
15834
|
+
raw = readFileSync(path, "utf8");
|
|
15835
|
+
if (Buffer.byteLength(raw) > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
|
|
15836
|
+
} catch (error) {
|
|
15837
|
+
return { ...emptyRecoveryJournal(), status: error?.code === "ENOENT" ? "missing" : "unavailable" };
|
|
15838
|
+
}
|
|
15839
|
+
try {
|
|
15840
|
+
const journal = JSON.parse(raw);
|
|
15841
|
+
const entries = journal.entries?.map(recoveryEvent);
|
|
15842
|
+
const current = journal.current && Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
15843
|
+
resource, journal.current[resource] === null ? null : recoveryEvent(journal.current[resource]),
|
|
15844
|
+
]));
|
|
15845
|
+
if (!exactKeys(journal, ["version", "current", "entries"]) || journal.version !== 1 ||
|
|
15846
|
+
!exactKeys(journal.current, RATE_RESOURCES) || !Array.isArray(journal.entries) ||
|
|
15847
|
+
journal.entries.length > RECOVERY_JOURNAL_MAX_ENTRIES ||
|
|
15848
|
+
entries.some((entry) => entry === null) ||
|
|
15849
|
+
RATE_RESOURCES.some((resource) => journal.current[resource] !== null &&
|
|
15850
|
+
(current[resource] === null || current[resource].code === "clear"))) {
|
|
15851
|
+
return { ...emptyRecoveryJournal(), status: "corrupt" };
|
|
15852
|
+
}
|
|
15853
|
+
return { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
|
|
15854
|
+
[resource, visibleRecoveryEvent(current[resource])])),
|
|
15855
|
+
entries: entries.map(visibleRecoveryEvent), status: "healthy" };
|
|
15856
|
+
} catch {
|
|
15857
|
+
return { ...emptyRecoveryJournal(), status: "corrupt" };
|
|
15858
|
+
}
|
|
15859
|
+
}
|
|
15860
|
+
|
|
15861
|
+
function recordRecoveryTransition(scope, event) {
|
|
15862
|
+
if (!scope?.path || !RATE_RESOURCES.includes(event?.resource)) return { ok: false, reason: "invalid" };
|
|
15863
|
+
const path = recoveryJournalPath(scope);
|
|
15864
|
+
return withPersistenceLock(path, () => {
|
|
15865
|
+
const previous = readRecoveryTransitions(scope);
|
|
15866
|
+
if (!["healthy", "missing"].includes(previous.status)) return { ok: false, reason: previous.status };
|
|
15867
|
+
const plain = (entry) => entry && Object.fromEntries(RECOVERY_EVENT_KEYS.map((key) => [key, entry[key]]));
|
|
15868
|
+
const journal = { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
|
|
15869
|
+
[resource, plain(previous.current[resource])])),
|
|
15870
|
+
entries: previous.entries.map((entry) => entry.code === "clear" ? entry : plain(entry)) };
|
|
15871
|
+
const former = journal.current[event.resource];
|
|
15872
|
+
const at = Number.isFinite(event.at) ? event.at : Date.now();
|
|
15873
|
+
if (event.clear === true) {
|
|
15874
|
+
if (!former) return { ok: true, changed: false };
|
|
15875
|
+
journal.current[event.resource] = null;
|
|
15876
|
+
journal.entries.push({ code: "clear", resource: event.resource, at });
|
|
15877
|
+
} else {
|
|
15878
|
+
const entry = plain(recoveryCause({ ...event, reason: event.reason ?? event.code, at }));
|
|
15879
|
+
entry.firstAt = former?.code === entry.code && former?.origin === entry.origin &&
|
|
15880
|
+
former?.tab === entry.tab ? Math.min(former.firstAt, at) : at;
|
|
15881
|
+
if (Buffer.byteLength(JSON.stringify(entry)) > RECOVERY_ENTRY_MAX_BYTES) return { ok: false, reason: "capacity" };
|
|
15882
|
+
journal.current[event.resource] = entry;
|
|
15883
|
+
if (former?.code !== entry.code || former?.origin !== entry.origin || former?.tab !== entry.tab) {
|
|
15884
|
+
journal.entries.push(entry);
|
|
15885
|
+
}
|
|
15886
|
+
}
|
|
15887
|
+
journal.entries = journal.entries.slice(-RECOVERY_JOURNAL_MAX_ENTRIES);
|
|
15888
|
+
while (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES && journal.entries.length > 0) {
|
|
15889
|
+
journal.entries.shift();
|
|
15890
|
+
}
|
|
15891
|
+
if (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES) return { ok: false, reason: "capacity" };
|
|
15892
|
+
const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
15893
|
+
try {
|
|
15894
|
+
writeFileSync(temp, `${JSON.stringify(journal)}\n`, { mode: 0o600, flag: "wx" });
|
|
15895
|
+
renameSync(temp, path);
|
|
15896
|
+
return { ok: true, changed: true };
|
|
15897
|
+
} catch (error) {
|
|
15898
|
+
try { unlinkSync(temp); } catch { /* only our temporary file */ }
|
|
15899
|
+
return { ok: false, reason: error?.code ?? "unwritable" };
|
|
15900
|
+
}
|
|
15901
|
+
});
|
|
15902
|
+
}
|
|
15903
|
+
|
|
15904
|
+
function safeRecordRecoveryTransition(scope, event, write = recordRecoveryTransition) {
|
|
15905
|
+
try { return write(scope, event); } catch { return { ok: false, reason: "unavailable" }; }
|
|
15906
|
+
}
|
|
15907
|
+
|
|
15908
|
+
function recoveryLedgerSummary(scope, resource, nowMs = Date.now()) {
|
|
15909
|
+
if (!scope?.path || !RATE_RESOURCES.includes(resource)) return null;
|
|
15910
|
+
const loaded = readGovernorState(scope.path, nowMs, { persistMigration: false });
|
|
15911
|
+
if (!loaded.ok) return null;
|
|
15912
|
+
const state = loaded.value;
|
|
15913
|
+
const debt = state.debt[resource];
|
|
15914
|
+
const debtUnits = debt.unresolvedUnits + debt.quiescentUnits;
|
|
15915
|
+
const charged = governorChargedCost(state, resource, nowMs);
|
|
15916
|
+
const oldest = Object.values(state.debtGroups)
|
|
15917
|
+
.filter((group) => !group.quiescent && group.units[resource] > 0)
|
|
15918
|
+
.map((group) => group.at);
|
|
15919
|
+
return {
|
|
15920
|
+
receiptUnits: Number.isSafeInteger(charged) ? charged - debtUnits : null,
|
|
15921
|
+
debtUnits,
|
|
15922
|
+
oldestUnresolvedAt: oldest.length > 0 ? Math.min(...oldest) : null,
|
|
15923
|
+
observerAt: state.observers[resource]?.at > 0 && state.observers[resource].at <= nowMs
|
|
15924
|
+
? state.observers[resource].at : null,
|
|
15925
|
+
debtCount: debt.unresolvedCount + debt.quiescentCount,
|
|
15926
|
+
receiptCount: Object.values(state.reservations).filter((reservation) =>
|
|
15927
|
+
reservationCost(reservation, resource, state.leases, nowMs) > 0).length +
|
|
15928
|
+
(reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs) > 0 ? 1 : 0),
|
|
15929
|
+
};
|
|
15930
|
+
}
|
|
15931
|
+
|
|
14459
15932
|
function acquisitionFailureCopy(reason) {
|
|
14460
15933
|
if (reason === "busy") return { short: "lock busy", narrow: "Shared lock busy", full: "Acquisition store is busy; checking the shared lock" };
|
|
14461
15934
|
if (reason === "unwritable") return { short: "store blocked", narrow: "Store blocked", full: "Acquisition store cannot be written; check config permissions" };
|
|
@@ -14557,25 +16030,6 @@ function measuredNestedOperationCost(operation, value) {
|
|
|
14557
16030
|
return actual;
|
|
14558
16031
|
}
|
|
14559
16032
|
|
|
14560
|
-
function uncertainReservationReceipts(scope, reservationId, nowMs) {
|
|
14561
|
-
const snapshot = inspectGovernor(scope, nowMs);
|
|
14562
|
-
const reservation = snapshot.ok ? snapshot.value.reservations?.[reservationId] : null;
|
|
14563
|
-
const accessKey = scope.accessKey ?? scope.hash;
|
|
14564
|
-
if (!reservation || typeof accessKey !== "string") return [];
|
|
14565
|
-
return RATE_RESOURCES.flatMap((resource) => {
|
|
14566
|
-
if (reservation.costs[resource] <= 0) return [];
|
|
14567
|
-
return [{
|
|
14568
|
-
id: `${reservationId}:${resource}`,
|
|
14569
|
-
reservationId,
|
|
14570
|
-
accessKey,
|
|
14571
|
-
resource,
|
|
14572
|
-
epoch: reservation.epochs[resource],
|
|
14573
|
-
units: reservation.costs[resource],
|
|
14574
|
-
startedAt: reservation.startedAt,
|
|
14575
|
-
}];
|
|
14576
|
-
});
|
|
14577
|
-
}
|
|
14578
|
-
|
|
14579
16033
|
function abortableDelay(ms, signal, {
|
|
14580
16034
|
setTimeout: setTimeout_ = setTimeout,
|
|
14581
16035
|
clearTimeout: clearTimeout_ = clearTimeout,
|
|
@@ -14770,30 +16224,38 @@ async function runAdmittedOperation({
|
|
|
14770
16224
|
}
|
|
14771
16225
|
const reservationId = admitted.value.reservationId;
|
|
14772
16226
|
const settlementScope = { ...scope, identityProvider: null };
|
|
14773
|
-
const
|
|
16227
|
+
const transportScope = { ...scope, receiptCapability: admitted.value.receiptCapability };
|
|
14774
16228
|
try {
|
|
14775
|
-
const value = await requestIdentityStorage.run(
|
|
16229
|
+
const value = await requestIdentityStorage.run(transportScope, () => run(signal));
|
|
14776
16230
|
const measured = measuredNestedOperationCost(operation, value);
|
|
14777
16231
|
const costs = measured ?? operationCost(operation);
|
|
14778
|
-
const outcome = value?.
|
|
14779
|
-
|
|
16232
|
+
const outcome = value?.notStarted === true ? "measured-success" :
|
|
16233
|
+
value?.ok === false ? governorOutcomeForError(value.failure) : "measured-success";
|
|
16234
|
+
const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
|
|
14780
16235
|
outcome,
|
|
14781
|
-
...(
|
|
14782
|
-
|
|
16236
|
+
...(value?.notStarted === true ? { neverStarted: true } : {}),
|
|
16237
|
+
...(measured || outcome === "measured-success" ? { actualCosts: costs } : {}),
|
|
16238
|
+
}, now);
|
|
16239
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
16240
|
+
return { ok: false, error: new Error(`API receipt settlement ${terminal.status}`),
|
|
16241
|
+
reservationId };
|
|
16242
|
+
}
|
|
14783
16243
|
const identityCoordinator = scope.identityCoordinator ?? runtimeIdentityCoordinator;
|
|
14784
16244
|
if (scope.accessKey && scope.accessKey !== identityCoordinator?.current()?.accessKey) {
|
|
14785
16245
|
return { ok: false, error: new Error("Credential changed"), reservationId };
|
|
14786
16246
|
}
|
|
14787
|
-
return { ok: true, value, reservationId
|
|
14788
|
-
uncertainReceipts: measured ? [] : pendingReceipts };
|
|
16247
|
+
return { ok: true, value, reservationId };
|
|
14789
16248
|
} catch (error) {
|
|
14790
16249
|
const actualCost = measuredNestedOperationCost(operation, { requestMetrics: error?.requestMetrics });
|
|
14791
|
-
const
|
|
14792
|
-
completeReservation(settlementScope, reservationId, {
|
|
16250
|
+
const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
|
|
14793
16251
|
outcome: governorOutcomeForError(error),
|
|
14794
|
-
...(
|
|
14795
|
-
|
|
14796
|
-
|
|
16252
|
+
...(error?.notStarted === true ? { neverStarted: true } : {}),
|
|
16253
|
+
...(actualCost ? { actualCosts: actualCost } : {}),
|
|
16254
|
+
}, now);
|
|
16255
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
16256
|
+
error.receiptSettlement = terminal.status;
|
|
16257
|
+
}
|
|
16258
|
+
return { ok: false, error, reservationId };
|
|
14797
16259
|
}
|
|
14798
16260
|
}
|
|
14799
16261
|
|
|
@@ -15010,6 +16472,7 @@ function refreshStatus({
|
|
|
15010
16472
|
stale = false,
|
|
15011
16473
|
disconnected = false,
|
|
15012
16474
|
screenReader = false,
|
|
16475
|
+
onDemand = false,
|
|
15013
16476
|
} = {}) {
|
|
15014
16477
|
const status = (kind, glyphKind, label, tone, animate = false, detailKind = null) => ({
|
|
15015
16478
|
kind,
|
|
@@ -15046,6 +16509,7 @@ function refreshStatus({
|
|
|
15046
16509
|
// Keep the cause visible until a successful result clears it.
|
|
15047
16510
|
if (activeError) return status("failed", "failed", "Failed", "attention");
|
|
15048
16511
|
if (securityIncomplete) return status("limited", "limited", "Limited", "attention");
|
|
16512
|
+
if (onDemand) return status("cached", "watching", "Cached", "inert");
|
|
15049
16513
|
if (sharedData) return status("shared", "watching", "Shared", "inert", false, "sharing");
|
|
15050
16514
|
if (["waiting", "pending", "probe"].includes(mode)) {
|
|
15051
16515
|
const sharing = sharedLaneProvenance(governorDecision).waitCause === "shared-lane";
|
|
@@ -15535,6 +16999,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15535
16999
|
const [waiting, setWaiting] = useState({ actions: true, issues: true, prs: true, security: true });
|
|
15536
17000
|
const [governorDecisions, setGovernorDecisions] = useState(probingGovernorDecisions);
|
|
15537
17001
|
const [acquisitionFailures, setAcquisitionFailures] = useState({});
|
|
17002
|
+
const [recoveryCauses, setRecoveryCauses] = useState({ core: null, graphql: null });
|
|
17003
|
+
const recoveryRef = useRef(recoveryCauses);
|
|
15538
17004
|
const [governorEpochs, setGovernorEpochs] = useState(null);
|
|
15539
17005
|
const [requestStatuses, setRequestStatuses] = useState(() =>
|
|
15540
17006
|
Object.fromEntries(TABS.map((candidate) => [candidate.key, null])),
|
|
@@ -15560,6 +17026,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15560
17026
|
? { mode: "paused", reason: activeFailure, coordinationError: true,
|
|
15561
17027
|
cause: acquisitionFailureCopy(activeFailure).short }
|
|
15562
17028
|
: governorDecisions[tab.key];
|
|
17029
|
+
const activeResource = RATE_RESOURCES.find((resource) => tabRequestCost(tab.key)[resource] > 0) ?? null;
|
|
17030
|
+
const retainedRecovery = activeResource ? recoveryCauses[activeResource] : null;
|
|
17031
|
+
const activeRecovery = retainedRecovery?.origin === "source" && retainedRecovery.tab !== tab.key
|
|
17032
|
+
? null : retainedRecovery;
|
|
15563
17033
|
const activeRequestStatus = requestStatuses[tab.key];
|
|
15564
17034
|
// Once any endpoint proves the folder has no remote, the whole dashboard is
|
|
15565
17035
|
// in setup mode. Keeping this tab-local made a quick tab switch replace the
|
|
@@ -15590,7 +17060,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15590
17060
|
// Counts the lines actually rendered, not the notes collected -- getting this
|
|
15591
17061
|
// wrong by one row is what makes ink repaint the whole frame.
|
|
15592
17062
|
const extraLines =
|
|
15593
|
-
|
|
17063
|
+
(activeRecovery || tabError || activeGovernorDecision?.coordinationError ||
|
|
17064
|
+
activeGovernorDecision?.mode === "paused" || activeGovernorDecision?.disconnected ||
|
|
17065
|
+
data[tab.key] !== null && !runtimeIdentityCoordinator?.current()
|
|
17066
|
+
? 3 : NOTICE_ROWS) +
|
|
15594
17067
|
(tab.key === "security" && !remoteSetup ? securityLines.length : 0);
|
|
15595
17068
|
// Reserve lines for: the tab bar and the divider under it (2), the panel's
|
|
15596
17069
|
// top and bottom edges (2), the column header and its separator (2), and
|
|
@@ -15641,6 +17114,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15641
17114
|
const ageMs = collectorCachedSourceAge(cachedTab);
|
|
15642
17115
|
return ageMs === null ? [] : [[candidate.key, { ageMs, measuredAt: performance.now() }]];
|
|
15643
17116
|
})));
|
|
17117
|
+
// Keep server source timestamps separate from client-local age estimates.
|
|
17118
|
+
// An SSH receipt's age is measured on the client and cannot prove a new
|
|
17119
|
+
// source observation when a cached snapshot is delivered again.
|
|
17120
|
+
const collectorSourceSuccessRef = useRef(Object.fromEntries(TABS.flatMap((candidate) => {
|
|
17121
|
+
const cachedTab = cachedEntry?.tabs[candidate.key];
|
|
17122
|
+
const sourceAt = cachedTab?.source?.lastSuccessAt ?? cachedTab?.lastOk;
|
|
17123
|
+
return Number.isFinite(sourceAt) ? [[candidate.key, sourceAt]] : [];
|
|
17124
|
+
})));
|
|
15644
17125
|
const fetchTabRef = useRef(null);
|
|
15645
17126
|
const contextCoordinatorRef = useRef(null);
|
|
15646
17127
|
const governorRef = useRef(null);
|
|
@@ -16106,6 +17587,53 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16106
17587
|
: { ...current, [key]: value });
|
|
16107
17588
|
}
|
|
16108
17589
|
|
|
17590
|
+
const scheduledControlAttempts = new Map();
|
|
17591
|
+
|
|
17592
|
+
function recordRuntimeRecovery(key, reason, { retryAt = null, origin = "coordination" } = {}) {
|
|
17593
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
|
|
17594
|
+
if (!resource) return;
|
|
17595
|
+
const at = Date.now();
|
|
17596
|
+
const previous = recoveryRef.current[resource];
|
|
17597
|
+
const candidate = recoveryCause({
|
|
17598
|
+
reason, resource, at, retryAt, sourceAt: lastOkRef.current[key] ?? null,
|
|
17599
|
+
origin, tab: origin === "source" ? key : null,
|
|
17600
|
+
});
|
|
17601
|
+
const hardFailure = ["corrupt", "unwritable", "capacity", "accounting-overflow"].includes(candidate.code);
|
|
17602
|
+
if (!hardFailure && previous?.code === "receipt-retry" &&
|
|
17603
|
+
terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
|
|
17604
|
+
if (!hardFailure && previous?.code === "request-queue" &&
|
|
17605
|
+
cleanupQueueForResource(cleanupQueue, resource) > 0) return;
|
|
17606
|
+
if (previous?.origin === "source" && previous.tab === key && origin === "coordination" &&
|
|
17607
|
+
!["corrupt", "unwritable", "capacity", "accounting-overflow",
|
|
17608
|
+
"receipt-retry", "interrupted-request"].includes(candidate.code)) return;
|
|
17609
|
+
if (previous?.code === candidate.code && previous?.retryAt === candidate.retryAt &&
|
|
17610
|
+
previous?.origin === candidate.origin && previous?.tab === candidate.tab &&
|
|
17611
|
+
at - previous.at < 60_000) return;
|
|
17612
|
+
const cause = recoveryCause({ ...candidate, reason: candidate.code,
|
|
17613
|
+
terminalizationBacklog: cleanupQueueForResource(cleanupQueue, resource) +
|
|
17614
|
+
terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource),
|
|
17615
|
+
...recoveryLedgerSummary(scope, resource, at) });
|
|
17616
|
+
cause.firstAt = previous?.code === cause.code && previous?.origin === cause.origin &&
|
|
17617
|
+
previous?.tab === cause.tab ? previous.firstAt : at;
|
|
17618
|
+
recoveryRef.current = { ...recoveryRef.current, [resource]: cause };
|
|
17619
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17620
|
+
if (scope) setTimeout(() => { safeRecordRecoveryTransition(scope, cause); }, 0).unref?.();
|
|
17621
|
+
}
|
|
17622
|
+
|
|
17623
|
+
function clearRuntimeRecovery(resource, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
|
|
17624
|
+
const previous = recoveryRef.current[resource];
|
|
17625
|
+
if (previous?.code === "receipt-retry" &&
|
|
17626
|
+
terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
|
|
17627
|
+
if (previous?.code === "request-queue" &&
|
|
17628
|
+
cleanupQueueForResource(cleanupQueue, resource) > 0) return;
|
|
17629
|
+
if (!shouldClearRecoveryCause(previous, { sourceSuccess, tab, scheduledHold })) return;
|
|
17630
|
+
recoveryRef.current = { ...recoveryRef.current, [resource]: null };
|
|
17631
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17632
|
+
if (scope) setTimeout(() => {
|
|
17633
|
+
safeRecordRecoveryTransition(scope, { resource, clear: true, at: Date.now() });
|
|
17634
|
+
}, 0).unref?.();
|
|
17635
|
+
}
|
|
17636
|
+
|
|
16109
17637
|
function publishGovernorEpochs(snapshot) {
|
|
16110
17638
|
const epochs = snapshot?.value?.epochs;
|
|
16111
17639
|
if (!isRecord(epochs)) return;
|
|
@@ -16135,7 +17663,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16135
17663
|
return null;
|
|
16136
17664
|
}
|
|
16137
17665
|
|
|
16138
|
-
function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false) {
|
|
17666
|
+
function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false, retryAt = null) {
|
|
17667
|
+
recordRuntimeRecovery(key, acquisitionFailure && reason === "busy" ? "acquisition-busy" : reason,
|
|
17668
|
+
{ retryAt });
|
|
16139
17669
|
if (acquisitionFailure) setAcquisitionFailures((current) => current[key] === reason
|
|
16140
17670
|
? current : { ...current, [key]: reason ?? "unavailable" });
|
|
16141
17671
|
const hold = diagnosticHold ?? (
|
|
@@ -16180,16 +17710,27 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16180
17710
|
return;
|
|
16181
17711
|
}
|
|
16182
17712
|
if (!snapshot?.ok) {
|
|
16183
|
-
pauseCoordination(key, snapshot?.reason);
|
|
17713
|
+
pauseCoordination(key, snapshot?.reason, null, false, snapshot?.retryAt);
|
|
16184
17714
|
return;
|
|
16185
17715
|
}
|
|
16186
17716
|
const tabReady = governorTabControlReady(snapshot, key, nowMs);
|
|
16187
17717
|
if (!refreshed?.ok && !tabReady) {
|
|
16188
|
-
|
|
17718
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0 &&
|
|
17719
|
+
snapshot.value.observers[candidate]?.outcome !== "healthy");
|
|
17720
|
+
pauseCoordination(key, refreshed?.reason, "observer", false,
|
|
17721
|
+
resource ? snapshot.value.observers[resource]?.nextAt : null);
|
|
16189
17722
|
return;
|
|
16190
17723
|
}
|
|
16191
17724
|
publishGovernorEpochs(snapshot);
|
|
16192
17725
|
if (refreshed.value?.status === "waiting" && !tabReady) {
|
|
17726
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
|
|
17727
|
+
if (resource && !recoveryRef.current[resource] && scope) {
|
|
17728
|
+
const shared = readRecoveryTransitions(scope).current[resource];
|
|
17729
|
+
if (shared?.origin === "coordination") {
|
|
17730
|
+
recoveryRef.current = { ...recoveryRef.current, [resource]: shared };
|
|
17731
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17732
|
+
}
|
|
17733
|
+
}
|
|
16193
17734
|
setTabGovernorDecision(key, { mode: "waiting", probing: true });
|
|
16194
17735
|
return;
|
|
16195
17736
|
}
|
|
@@ -16198,32 +17739,45 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16198
17739
|
const reservation = pendingIntent
|
|
16199
17740
|
? snapshot.value.reservations[`reservation:${pendingIntent.intentId}`]
|
|
16200
17741
|
: null;
|
|
17742
|
+
const costs = tabRequestCost(key);
|
|
16201
17743
|
if (reservation?.status === "scheduled" && reservation.notBefore > nowMs) {
|
|
17744
|
+
const accessKey = pendingIntent.cleanupScope?.accessKey;
|
|
17745
|
+
const reservationId = `reservation:${pendingIntent.intentId}`;
|
|
17746
|
+
const lastAttempt = scheduledControlAttempts.get(key);
|
|
17747
|
+
if (accessKey && pendingIntent.cleanupScope?.hash === scope?.hash &&
|
|
17748
|
+
identity()?.accessKey === accessKey &&
|
|
17749
|
+
(lastAttempt?.id !== reservationId || nowMs - lastAttempt.at >= 30_000)) {
|
|
17750
|
+
scheduledControlAttempts.set(key, { id: reservationId, at: nowMs });
|
|
17751
|
+
// A guarded hold write can retire an older transient notice. Merely
|
|
17752
|
+
// reading a future grant or an absent hold cannot prove recovery.
|
|
17753
|
+
const cleared = clearScheduledRuntimeControlHold(acquisitionEngine,
|
|
17754
|
+
acquisitionSubscriptions, key, accessKey, nowMs);
|
|
17755
|
+
for (const resource of RATE_RESOURCES) {
|
|
17756
|
+
if (costs[resource] > 0) clearRuntimeRecovery(resource, { scheduledHold: cleared });
|
|
17757
|
+
}
|
|
17758
|
+
}
|
|
16202
17759
|
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
16203
17760
|
status: "scheduled",
|
|
16204
|
-
reservationId
|
|
17761
|
+
reservationId,
|
|
16205
17762
|
...reservation,
|
|
16206
17763
|
...currentSharedLaneProvenance(pendingIntent, snapshot.value.leases, nowMs),
|
|
16207
17764
|
}));
|
|
16208
17765
|
return;
|
|
16209
17766
|
}
|
|
16210
|
-
const costs = tabRequestCost(key);
|
|
16211
17767
|
for (const resource of RATE_RESOURCES) {
|
|
16212
17768
|
if (costs[resource] <= 0) continue;
|
|
16213
17769
|
const budget = snapshot.value.budgets[resource];
|
|
16214
17770
|
const decision = budget
|
|
16215
|
-
?
|
|
16216
|
-
budget,
|
|
16217
|
-
resource,
|
|
16218
|
-
leases: snapshot.value.leases,
|
|
16219
|
-
reservations: Object.values(snapshot.value.reservations),
|
|
16220
|
-
nowMs,
|
|
16221
|
-
cost: costs[resource],
|
|
16222
|
-
})
|
|
17771
|
+
? governorResourceDecision(snapshot.value, resource, nowMs, costs[resource])
|
|
16223
17772
|
: { mode: "probe", reason: "budget-unknown" };
|
|
16224
17773
|
if (decision.mode === "open") continue;
|
|
16225
17774
|
const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision.reason) ||
|
|
16226
17775
|
budget?.blockUntil > nowMs;
|
|
17776
|
+
const holdReason = budget?.blockUntil > nowMs
|
|
17777
|
+
? budget.blockReason ?? decision.reason : decision.reason;
|
|
17778
|
+
recordRuntimeRecovery(key, holdReason, {
|
|
17779
|
+
retryAt: recoveryRetryAt(decision, budget, snapshot.value.observers[resource], nowMs),
|
|
17780
|
+
});
|
|
16227
17781
|
setRuntimeAcquisitionHold(
|
|
16228
17782
|
acquisitionEngine,
|
|
16229
17783
|
acquisitionSubscriptions,
|
|
@@ -16235,7 +17789,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16235
17789
|
);
|
|
16236
17790
|
setTabGovernorDecision(key, {
|
|
16237
17791
|
mode: resetHold || decision.mode === "paused" ? "paused" : "waiting",
|
|
16238
|
-
reason:
|
|
17792
|
+
reason: holdReason,
|
|
16239
17793
|
resetMs: budget?.resetMs ?? decision.resetMs ?? null,
|
|
16240
17794
|
probing: decision.mode === "probe" && !resetHold,
|
|
16241
17795
|
});
|
|
@@ -16249,6 +17803,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16249
17803
|
identity()?.accessKey ?? null,
|
|
16250
17804
|
);
|
|
16251
17805
|
setTabGovernorDecision(key, null);
|
|
17806
|
+
for (const resource of RATE_RESOURCES) {
|
|
17807
|
+
if (costs[resource] > 0) clearRuntimeRecovery(resource);
|
|
17808
|
+
}
|
|
16252
17809
|
}
|
|
16253
17810
|
|
|
16254
17811
|
async function refreshRuntimeBudget(currentScope) {
|
|
@@ -16260,15 +17817,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16260
17817
|
return refreshed;
|
|
16261
17818
|
}
|
|
16262
17819
|
|
|
16263
|
-
function
|
|
17820
|
+
function syncRuntimeQuotaProjection(currentScope, snapshot) {
|
|
16264
17821
|
if (!snapshot?.ok) return;
|
|
16265
|
-
|
|
16266
|
-
const observer = snapshot.value.observers?.[resource];
|
|
16267
|
-
const epoch = snapshot.value.epochs?.[resource];
|
|
16268
|
-
return observer?.outcome === "healthy" && typeof epoch === "string" && Number.isFinite(observer.at)
|
|
16269
|
-
? [[resource, { epoch, observedAt: observer.at }]] : [];
|
|
16270
|
-
}));
|
|
16271
|
-
acquisitionEngine.reconcileUncertainty(currentScope.accessKey, evidence);
|
|
17822
|
+
acquisitionEngine.syncQuotaProjection(currentScope);
|
|
16272
17823
|
}
|
|
16273
17824
|
|
|
16274
17825
|
function commit(key, run, {
|
|
@@ -16320,7 +17871,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16320
17871
|
: run;
|
|
16321
17872
|
return requestIdentityStorage.run(scope, fencedRun)
|
|
16322
17873
|
.then(async (result) => {
|
|
16323
|
-
|
|
17874
|
+
const terminal = queueTerminalization(settlementScope, scope.receiptCapability, result?.measuredSuccess === false
|
|
16324
17875
|
? { outcome: "rejected", observations: result?.observations ?? [] }
|
|
16325
17876
|
: {
|
|
16326
17877
|
outcome: "measured-success",
|
|
@@ -16329,7 +17880,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16329
17880
|
graphql: result?.graphqlSpent ?? GRAPHQL_PER_FETCH[key] ?? 0,
|
|
16330
17881
|
},
|
|
16331
17882
|
observations: result?.observations ?? [],
|
|
16332
|
-
}
|
|
17883
|
+
});
|
|
17884
|
+
if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
|
|
17885
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
17886
|
+
failAcquisition(acquisition, { requestMetrics: acquisitionRequestMetrics(result), hold: "primary" });
|
|
17887
|
+
pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
|
|
17888
|
+
return;
|
|
17889
|
+
}
|
|
16333
17890
|
if (!currentAccess()) {
|
|
16334
17891
|
failAcquisition(acquisition, {
|
|
16335
17892
|
requestMetrics: acquisitionRequestMetrics(result),
|
|
@@ -16352,7 +17909,6 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16352
17909
|
acquisition.capabilities = result.capabilities ?? {};
|
|
16353
17910
|
acquisition.repositoryIdentity = result.repositoryIdentity;
|
|
16354
17911
|
acquisition.requestMetrics = acquisitionRequestMetrics(result);
|
|
16355
|
-
acquisition.uncertainReceipts = result.uncertainReceipts ?? [];
|
|
16356
17912
|
}
|
|
16357
17913
|
// Identical payload: skip the parse *and* the state update. Returning
|
|
16358
17914
|
// the same state object makes React bail out of the re-render, so an
|
|
@@ -16387,9 +17943,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16387
17943
|
// so the next admitted tick parses its response instead of taking the
|
|
16388
17944
|
// identical-output fast path.
|
|
16389
17945
|
if (transition.kind === "unusable") {
|
|
17946
|
+
recordRuntimeRecovery(key, "source-incomplete", { origin: "source" });
|
|
16390
17947
|
failAcquisition(acquisition, {
|
|
16391
17948
|
requestMetrics: acquisition?.requestMetrics ?? {},
|
|
16392
|
-
uncertainReceipts: acquisition?.uncertainReceipts ?? [],
|
|
16393
17949
|
hold: "primary",
|
|
16394
17950
|
});
|
|
16395
17951
|
rawRef.current[key] = transition.nextRaw;
|
|
@@ -16402,9 +17958,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16402
17958
|
// freshness. Do not retain its raw value either, so the next source
|
|
16403
17959
|
// retry is parsed instead of taking the identical-payload fast path.
|
|
16404
17960
|
if (transition.kind === "blind") {
|
|
17961
|
+
recordRuntimeRecovery(key, "security-incomplete", { origin: "source" });
|
|
16405
17962
|
failAcquisition(acquisition, {
|
|
16406
17963
|
requestMetrics: acquisition?.requestMetrics ?? {},
|
|
16407
|
-
uncertainReceipts: acquisition?.uncertainReceipts ?? [],
|
|
16408
17964
|
hold: "primary",
|
|
16409
17965
|
});
|
|
16410
17966
|
rawRef.current[key] = transition.nextRaw;
|
|
@@ -16439,6 +17995,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16439
17995
|
publishStagedEntities(entityRef.current, stagedEntities, transition.kind);
|
|
16440
17996
|
if (transition.kind === "unchanged") {
|
|
16441
17997
|
lastOkRef.current[key] = completedAt;
|
|
17998
|
+
for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
|
|
17999
|
+
clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
|
|
18000
|
+
}
|
|
16442
18001
|
clearBackoff(`tab:${key}`);
|
|
16443
18002
|
setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
|
|
16444
18003
|
const cachedTab = pick(
|
|
@@ -16455,6 +18014,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16455
18014
|
const tabData = transition.data;
|
|
16456
18015
|
const tabMeta = transition.meta;
|
|
16457
18016
|
lastOkRef.current[key] = completedAt;
|
|
18017
|
+
for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
|
|
18018
|
+
clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
|
|
18019
|
+
}
|
|
16458
18020
|
clearBackoff(`tab:${key}`);
|
|
16459
18021
|
rawRef.current[key] = transition.nextRaw;
|
|
16460
18022
|
setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
|
|
@@ -16474,10 +18036,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16474
18036
|
requestMetrics: err?.requestMetrics ?? {},
|
|
16475
18037
|
hold: acquisitionFailureHold(err),
|
|
16476
18038
|
});
|
|
16477
|
-
|
|
18039
|
+
const terminal = queueTerminalization(settlementScope, scope.receiptCapability, {
|
|
16478
18040
|
outcome: governorOutcomeForError(err),
|
|
16479
18041
|
observations: err?.budgetObservations ?? [],
|
|
16480
|
-
}
|
|
18042
|
+
});
|
|
18043
|
+
if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
|
|
18044
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
18045
|
+
pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
|
|
18046
|
+
}
|
|
16481
18047
|
if (cancelled || !currentAccess() || err?.name === "AbortError") return;
|
|
16482
18048
|
// Preserve both the verdict and the bounded raw error in state. The
|
|
16483
18049
|
// renderer translates recognized verdicts at draw time, which lets a
|
|
@@ -16494,6 +18060,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16494
18060
|
// recover on the very next tick once the network is back.
|
|
16495
18061
|
const steps = verdict === "rate-limited" ? null : pick(FAILURE_LADDER, verdict, null);
|
|
16496
18062
|
if (steps) recordFailure(`tab:${key}`, performance.now(), steps);
|
|
18063
|
+
const backoffUntil = alertBackoff.get(backoffStorageKey(`tab:${key}`))?.until;
|
|
18064
|
+
const retryAt = Number.isFinite(backoffUntil)
|
|
18065
|
+
? Date.now() + Math.max(0, backoffUntil - performance.now()) : null;
|
|
18066
|
+
if (terminal.status !== "blocked" && terminal.status !== "compacted") {
|
|
18067
|
+
const recovery = requestFailureRecovery(err, verdict);
|
|
18068
|
+
recordRuntimeRecovery(key, recovery.reason, { origin: recovery.origin, retryAt });
|
|
18069
|
+
}
|
|
16497
18070
|
if (verdict === "rate-limited") {
|
|
16498
18071
|
const snapshot = inspectGovernor(scope, Date.now());
|
|
16499
18072
|
const costs = tabRequestCost(key);
|
|
@@ -16508,6 +18081,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16508
18081
|
});
|
|
16509
18082
|
}
|
|
16510
18083
|
handleRateLimitBlocks(key, blocks, Date.now());
|
|
18084
|
+
const held = inspectGovernor(scope, Date.now());
|
|
18085
|
+
const heldUntil = RATE_RESOURCES.filter((resource) => costs[resource] > 0)
|
|
18086
|
+
.map((resource) => held.value?.budgets?.[resource]?.blockUntil)
|
|
18087
|
+
.filter((deadline) => Number.isFinite(deadline) && deadline > Date.now());
|
|
18088
|
+
if (heldUntil.length > 0) recordRuntimeRecovery(key, "rate-limited", {
|
|
18089
|
+
origin: "source", retryAt: Math.min(...heldUntil),
|
|
18090
|
+
});
|
|
16511
18091
|
}
|
|
16512
18092
|
})
|
|
16513
18093
|
.finally(() => {
|
|
@@ -16541,6 +18121,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16541
18121
|
const leaseId = governorId();
|
|
16542
18122
|
const pending = new Map();
|
|
16543
18123
|
const cleanupQueue = new Set();
|
|
18124
|
+
let lastWallAt = Date.now();
|
|
16544
18125
|
// One manual handoff per tab, carrying which key produced it. A keypress
|
|
16545
18126
|
// during an automatic request must not be lost, but repeated keypresses
|
|
16546
18127
|
// during the manual request itself must not create a trailing second batch.
|
|
@@ -16579,12 +18160,20 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16579
18160
|
const unchangedPolls = Object.fromEntries(TAB_KEYS.map((key) => [key, 0]));
|
|
16580
18161
|
let pendingBlockPublications = new Map();
|
|
16581
18162
|
const acquisitionSubscriptions = new Map();
|
|
18163
|
+
// One-shot demand survives admission delays, but ends when an observation arrives.
|
|
18164
|
+
const requestedTabs = new Map();
|
|
16582
18165
|
|
|
16583
18166
|
function adoptAcquisitionSnapshot(key, snapshot, receipt = null) {
|
|
16584
18167
|
if (!snapshot || cancelled) return;
|
|
18168
|
+
if (requestedTabs.has(key) && !queuedManual.has(key) &&
|
|
18169
|
+
snapshot.lastSuccessAt >= requestedTabs.get(key)) {
|
|
18170
|
+
requestedTabs.delete(key);
|
|
18171
|
+
if (runtime.background === "on-demand" && key !== "actions") pollDueAt[key] = Infinity;
|
|
18172
|
+
}
|
|
16585
18173
|
if (runtime.connect === null && Number.isFinite(snapshot.nextDueAt)) {
|
|
16586
18174
|
sharedSnapshotDueAt = { ...sharedSnapshotDueAt, [key]: snapshot.nextDueAt };
|
|
16587
|
-
if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key
|
|
18175
|
+
if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key &&
|
|
18176
|
+
Number.isFinite(pollIntervalFor(key, key))) {
|
|
16588
18177
|
const at = Date.now();
|
|
16589
18178
|
const due = sharedAcquisitionPollDeadline(pollDueAt[key], snapshot.nextDueAt, at, true);
|
|
16590
18179
|
if (due < pollDueAt[key]) {
|
|
@@ -16657,6 +18246,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16657
18246
|
const host = effectiveRuntimeHost({ remoteUrls: runtimeRemoteUrls });
|
|
16658
18247
|
const handles = new Map();
|
|
16659
18248
|
const receivedSnapshots = new Set();
|
|
18249
|
+
const initialKey = TABS[activeIndexRef.current].key;
|
|
18250
|
+
const collectorDemand = (key) => ({
|
|
18251
|
+
active: TABS[activeIndexRef.current].key === key &&
|
|
18252
|
+
(runtime.background !== "on-demand" || key === "actions"),
|
|
18253
|
+
background: runtime.background === "all",
|
|
18254
|
+
floorMs: runtime.refreshMs,
|
|
18255
|
+
pages: pageStateRef.current[key]?.pages ?? 1,
|
|
18256
|
+
});
|
|
16660
18257
|
let collectorReady = false;
|
|
16661
18258
|
setGovernorDecisions(Object.fromEntries(TAB_KEYS.map((key) => [key, { disconnected: true }])));
|
|
16662
18259
|
if (!repository || !host) {
|
|
@@ -16686,39 +18283,46 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16686
18283
|
},
|
|
16687
18284
|
});
|
|
16688
18285
|
for (const key of TAB_KEYS) {
|
|
16689
|
-
const demand =
|
|
16690
|
-
active: TABS[activeIndexRef.current].key === key,
|
|
16691
|
-
background: runtime.background !== "off",
|
|
16692
|
-
floorMs: runtime.refreshMs,
|
|
16693
|
-
pages: pageStateRef.current[key]?.pages ?? 1,
|
|
16694
|
-
};
|
|
18286
|
+
const demand = collectorDemand(key);
|
|
16695
18287
|
const subscribed = client.subscribe({ host, repo: repository, resource: key, demand,
|
|
16696
18288
|
sourceCheckpoint: cachedEntry?.tabs[key]?.source ?? null,
|
|
16697
18289
|
onSnapshot: (snapshot, receipt) => {
|
|
16698
18290
|
receivedSnapshots.add(key);
|
|
18291
|
+
const sourceAdvanced = collectorSourceAdvanced(
|
|
18292
|
+
collectorSourceSuccessRef.current[key], snapshot.lastSuccessAt);
|
|
16699
18293
|
adoptAcquisitionSnapshot(key, snapshot, receipt);
|
|
18294
|
+
if (Number.isFinite(snapshot.lastSuccessAt) &&
|
|
18295
|
+
(!Number.isFinite(collectorSourceSuccessRef.current[key]) ||
|
|
18296
|
+
snapshot.lastSuccessAt > collectorSourceSuccessRef.current[key])) {
|
|
18297
|
+
collectorSourceSuccessRef.current[key] = snapshot.lastSuccessAt;
|
|
18298
|
+
}
|
|
18299
|
+
if (sourceAdvanced) {
|
|
18300
|
+
for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
|
|
18301
|
+
clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
|
|
18302
|
+
}
|
|
18303
|
+
}
|
|
16700
18304
|
setErrors((current) => current[key] === null ? current : { ...current, [key]: null });
|
|
16701
18305
|
setWaiting((current) => current[key] ? { ...current, [key]: false } : current);
|
|
16702
18306
|
setGovernorDecisions((current) => ({ ...current, [key]:
|
|
16703
18307
|
collectorDisplayDecision({ connected: collectorReady, hasSnapshot: true }) }));
|
|
16704
18308
|
},
|
|
16705
|
-
onHold: (hold) =>
|
|
16706
|
-
|
|
16707
|
-
|
|
18309
|
+
onHold: (hold) => {
|
|
18310
|
+
if (hold === "app-auth") recordRuntimeRecovery(key, "app-auth", { origin: "source" });
|
|
18311
|
+
setGovernorDecisions((current) => ({ ...current, [key]:
|
|
18312
|
+
collectorDisplayDecision({ connected: collectorReady,
|
|
18313
|
+
hasSnapshot: receivedSnapshots.has(key), hold }) }));
|
|
18314
|
+
},
|
|
16708
18315
|
});
|
|
16709
18316
|
if (subscribed.ok) handles.set(key, subscribed.value);
|
|
16710
18317
|
}
|
|
18318
|
+
if (runtime.background === "on-demand" && initialKey !== "actions") {
|
|
18319
|
+
handles.get(initialKey)?.refresh();
|
|
18320
|
+
}
|
|
16711
18321
|
fetchTabRef.current = (key, request = {}) => {
|
|
16712
18322
|
const handle = handles.get(key);
|
|
16713
18323
|
if (!handle) return;
|
|
16714
|
-
|
|
16715
|
-
|
|
16716
|
-
candidateHandle.updateDemand({ active: candidate === key, background: runtime.background !== "off",
|
|
16717
|
-
floorMs: runtime.refreshMs, pages: pageStateRef.current[candidate]?.pages ?? 1 });
|
|
16718
|
-
}
|
|
16719
|
-
} else {
|
|
16720
|
-
handle.updateDemand({ active: true, background: runtime.background !== "off",
|
|
16721
|
-
floorMs: runtime.refreshMs, pages: pageStateRef.current[key]?.pages ?? 1 });
|
|
18324
|
+
for (const [candidate, candidateHandle] of handles) {
|
|
18325
|
+
candidateHandle.updateDemand(collectorDemand(candidate));
|
|
16722
18326
|
}
|
|
16723
18327
|
handle.refresh(request.force === true);
|
|
16724
18328
|
};
|
|
@@ -16795,12 +18399,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16795
18399
|
demand: acquisition.demand?.active ? "active" : "inactive",
|
|
16796
18400
|
unchangedCount: publicationView.unchangedCount,
|
|
16797
18401
|
inProgressCI: key === "actions" && actionsInProgress(rows),
|
|
16798
|
-
background: runtime.background,
|
|
18402
|
+
background: runtime.background === "on-demand" ? "all" : runtime.background,
|
|
16799
18403
|
}),
|
|
16800
18404
|
hold: null,
|
|
16801
18405
|
capabilities: acquisition.capabilities ?? {},
|
|
16802
18406
|
requestMetrics: acquisition.requestMetrics ?? {},
|
|
16803
|
-
uncertainReceipts: acquisition.uncertainReceipts ?? [],
|
|
16804
18407
|
repositoryIdentity: acquisition.repositoryIdentity,
|
|
16805
18408
|
meta,
|
|
16806
18409
|
securityNotes: key === "security"
|
|
@@ -16841,10 +18444,17 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16841
18444
|
automaticStatusVisible) {
|
|
16842
18445
|
const descriptor = tabForKey(key);
|
|
16843
18446
|
let acquisitionStartAttempted = false;
|
|
18447
|
+
let transportScope = null;
|
|
16844
18448
|
const transported = await runStartedAcquisitionTransport(
|
|
16845
18449
|
() => {
|
|
16846
18450
|
const admitted = inspectGovernor(currentScope, nowMs);
|
|
16847
18451
|
if (!admitted.ok) return admitted;
|
|
18452
|
+
const admittedReservation = admitted.value.reservations[reservationId];
|
|
18453
|
+
if (!admittedReservation?.receipt || admittedReservation.status !== "started") {
|
|
18454
|
+
return { ok: false, reason: "compacted" };
|
|
18455
|
+
}
|
|
18456
|
+
transportScope = { ...currentScope,
|
|
18457
|
+
receiptCapability: receiptCapabilityFromReservation(reservationId, admittedReservation) };
|
|
16848
18458
|
acquisitionStartAttempted = true;
|
|
16849
18459
|
const receipt = {
|
|
16850
18460
|
reservationId,
|
|
@@ -16875,13 +18485,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16875
18485
|
force: item.force,
|
|
16876
18486
|
catalog: workflowCatalogRef.current,
|
|
16877
18487
|
pages: item.acquisition.demand?.pages ?? pageStateRef.current[key]?.pages ?? 1,
|
|
16878
|
-
governor: { scope:
|
|
18488
|
+
governor: { scope: transportScope, leaseId },
|
|
16879
18489
|
previousRaw: rawRef.current[key] ?? null,
|
|
16880
18490
|
}),
|
|
16881
18491
|
{
|
|
16882
18492
|
force: item.force,
|
|
16883
18493
|
manual: item.manual === true,
|
|
16884
|
-
scope:
|
|
18494
|
+
scope: transportScope,
|
|
16885
18495
|
reservationId,
|
|
16886
18496
|
admittedAt: nowMs,
|
|
16887
18497
|
automaticStatusVisible,
|
|
@@ -16892,6 +18502,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16892
18502
|
},
|
|
16893
18503
|
);
|
|
16894
18504
|
if (transported.ok) return transported;
|
|
18505
|
+
if (transportScope?.receiptCapability) {
|
|
18506
|
+
const terminal = queueTerminalization(currentScope,
|
|
18507
|
+
transportScope.receiptCapability, { outcome: "rejected", neverStarted: true });
|
|
18508
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
18509
|
+
pauseCoordination(key, terminal.reason ?? terminal.status, null, true);
|
|
18510
|
+
}
|
|
18511
|
+
}
|
|
16895
18512
|
pauseCoordination(key, transported.reason, null, acquisitionStartAttempted);
|
|
16896
18513
|
// The governor reservation is already started. Even a transient failed
|
|
16897
18514
|
// claim write cannot safely rebind it; retain its charge and get a fresh
|
|
@@ -16915,6 +18532,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16915
18532
|
demand: key === activeKey ? "active" : "inactive",
|
|
16916
18533
|
unchangedCount: unchangedPolls[key],
|
|
16917
18534
|
inProgressCI: key === "actions" && actionsInProgress(actionRows),
|
|
18535
|
+
requested: requestedTabs.has(key),
|
|
16918
18536
|
background: runtime.background,
|
|
16919
18537
|
});
|
|
16920
18538
|
}
|
|
@@ -16925,7 +18543,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16925
18543
|
const localDueAt = Number.isFinite(step) ? at + step : Number.POSITIVE_INFINITY;
|
|
16926
18544
|
pollDueAt = {
|
|
16927
18545
|
...pollDueAt,
|
|
16928
|
-
[key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active),
|
|
18546
|
+
[key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active && Number.isFinite(step)),
|
|
16929
18547
|
};
|
|
16930
18548
|
}
|
|
16931
18549
|
|
|
@@ -16937,6 +18555,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16937
18555
|
function openPollDeadlines(at) {
|
|
16938
18556
|
const activeKey = TABS[activeIndexRef.current].key;
|
|
16939
18557
|
const opened = { ...NEVER_DUE };
|
|
18558
|
+
if (runtime.background === "on-demand") requestedTabs.set(activeKey, at);
|
|
16940
18559
|
opened[activeKey] = at + 1;
|
|
16941
18560
|
let slot = 1;
|
|
16942
18561
|
for (const key of TAB_KEYS) {
|
|
@@ -17038,6 +18657,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17038
18657
|
coordinator.invalidate();
|
|
17039
18658
|
setFailureContext(null);
|
|
17040
18659
|
if (discardData) {
|
|
18660
|
+
recoveryRef.current = { core: null, graphql: null };
|
|
18661
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17041
18662
|
rawRef.current = {};
|
|
17042
18663
|
entityRef.current.clear();
|
|
17043
18664
|
lastOkRef.current = {};
|
|
@@ -17115,6 +18736,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17115
18736
|
}
|
|
17116
18737
|
scope = createQuotaScope(current, { root: runtimeIdentityCoordinator.root, identityProvider: identity });
|
|
17117
18738
|
cleanupScope = { ...scope, identityProvider: null };
|
|
18739
|
+
const savedRecovery = readRecoveryTransitions(scope);
|
|
18740
|
+
recoveryRef.current = savedRecovery.status === "healthy"
|
|
18741
|
+
? { ...savedRecovery.current } : { core: null, graphql: null };
|
|
18742
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17118
18743
|
const nextTarget = dashboardCacheTarget({ repo: runtime.repo, ghRepo: process.env.GH_REPO, host: current.host, account: current.accessKey });
|
|
17119
18744
|
// The mount-time target had to guess the account, and guesses
|
|
17120
18745
|
// "unverified" whenever the bounded warm resolution above did not land
|
|
@@ -17176,6 +18801,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17176
18801
|
function finishPendingCleanup(key, item) {
|
|
17177
18802
|
if (pending.get(key) === item) pending.delete(key);
|
|
17178
18803
|
cleanupQueue.delete(item);
|
|
18804
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
|
|
18805
|
+
if (resource && cleanupQueueForResource(cleanupQueue, resource) === 0 &&
|
|
18806
|
+
recoveryRef.current[resource]?.code === "request-queue") {
|
|
18807
|
+
clearRuntimeRecovery(resource);
|
|
18808
|
+
}
|
|
17179
18809
|
setAcquisitionFailures((current) => {
|
|
17180
18810
|
if (!Object.hasOwn(current, key)) return current;
|
|
17181
18811
|
const next = { ...current };
|
|
@@ -17204,7 +18834,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17204
18834
|
}
|
|
17205
18835
|
|
|
17206
18836
|
function retryPendingCleanup(key, item, at = Date.now()) {
|
|
17207
|
-
if (item.cleanupInFlight)
|
|
18837
|
+
if (item.cleanupInFlight) {
|
|
18838
|
+
recordRuntimeRecovery(key, "request-queue");
|
|
18839
|
+
return false;
|
|
18840
|
+
}
|
|
17208
18841
|
item.cleanupPending = true;
|
|
17209
18842
|
item.cleanupKey = key;
|
|
17210
18843
|
cleanupQueue.add(item);
|
|
@@ -17230,6 +18863,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17230
18863
|
finishPendingCleanup(key, item);
|
|
17231
18864
|
}
|
|
17232
18865
|
if (!item.governorClean) armWake("data", governorControlRetryAt(at, runtime.refreshMs), dataWake);
|
|
18866
|
+
if (pending.has(key)) recordRuntimeRecovery(key, "request-queue");
|
|
17233
18867
|
return !pending.has(key);
|
|
17234
18868
|
}
|
|
17235
18869
|
|
|
@@ -17252,7 +18886,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17252
18886
|
// the tab's failure ladder. `force` is the separate, stronger request that
|
|
17253
18887
|
// also drops validators -- the `R` key. `r` is manual and not forced, which
|
|
17254
18888
|
// is what makes a quiet refresh cost nothing.
|
|
17255
|
-
async function requestTab(key, kind = "active", { force = false } = {}) {
|
|
18889
|
+
async function requestTab(key, kind = "active", { force = false, detachTransport = false } = {}) {
|
|
17256
18890
|
const manual = kind === "manual";
|
|
17257
18891
|
const signal = controller.signal;
|
|
17258
18892
|
const monotonicNow = performance.now();
|
|
@@ -17333,7 +18967,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17333
18967
|
currentScope.accessKey,
|
|
17334
18968
|
);
|
|
17335
18969
|
if (ownership.value.role !== "producer") {
|
|
17336
|
-
if (ownership.value.snapshot)
|
|
18970
|
+
if (ownership.value.snapshot) {
|
|
18971
|
+
if (ownership.value.reason === "fresh") requestedTabs.delete(key);
|
|
18972
|
+
adoptAcquisitionSnapshot(key, ownership.value.snapshot);
|
|
18973
|
+
}
|
|
17337
18974
|
if (ownership.value.sharingCount > 1) {
|
|
17338
18975
|
setTabGovernorDecision(key, {
|
|
17339
18976
|
mode: "waiting",
|
|
@@ -17457,9 +19094,19 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17457
19094
|
setTabWaiting(key, false);
|
|
17458
19095
|
const item = pending.get(key);
|
|
17459
19096
|
if (!item) return { persisted: false, retry: false, kind, key };
|
|
17460
|
-
|
|
19097
|
+
const transport = startPendingAcquisitionTransport(
|
|
17461
19098
|
key, item, currentScope, decision.reservationId, nowMs, signal, false,
|
|
17462
19099
|
);
|
|
19100
|
+
if (detachTransport) {
|
|
19101
|
+
// The scheduling single-flight owns admission, not the HTTP lifetime.
|
|
19102
|
+
// A background response must not block the next active-tab wake.
|
|
19103
|
+
void transport.catch((error) => {
|
|
19104
|
+
if (cancelled || pending.get(key) !== item) return;
|
|
19105
|
+
item.retryOnCleanup = true;
|
|
19106
|
+
cancelPending(key, currentScope);
|
|
19107
|
+
pauseCoordination(key, error?.reason ?? "stale");
|
|
19108
|
+
});
|
|
19109
|
+
} else await transport;
|
|
17463
19110
|
return { persisted: true, retry: false, kind, key };
|
|
17464
19111
|
}
|
|
17465
19112
|
fetchTabRef.current = (key, { force = false, kind = force ? "manual" : "active" } = {}) => {
|
|
@@ -17470,6 +19117,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17470
19117
|
unchangedPolls[key] = advanceUnchangedCount(unchangedPolls[key], "subscribed");
|
|
17471
19118
|
}
|
|
17472
19119
|
const requestedAt = Date.now();
|
|
19120
|
+
if (runtime.background === "on-demand") {
|
|
19121
|
+
if (kind === "tab-switch") requestedTabs.clear();
|
|
19122
|
+
requestedTabs.set(key, requestedAt);
|
|
19123
|
+
}
|
|
17473
19124
|
// Manual work replaces the active poll that would otherwise be due. Move
|
|
17474
19125
|
// that deadline before touching the governor so a data wake already in
|
|
17475
19126
|
// this event-loop turn cannot start a second batch as soon as the forced
|
|
@@ -17583,11 +19234,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17583
19234
|
const budget = snapshot.value.budgets[resource];
|
|
17584
19235
|
const observer = snapshot.value.observers[resource];
|
|
17585
19236
|
const decision = budget
|
|
17586
|
-
?
|
|
19237
|
+
? governorAvailableForGrant(snapshot.value, resource, nowMs)
|
|
17587
19238
|
: { mode: "paused" };
|
|
17588
|
-
const held =
|
|
17589
|
-
|
|
17590
|
-
decision.mode === "paused" && decision.reason === "budget-exhausted";
|
|
19239
|
+
const held = governorResourceHeld(snapshot.value, resource, nowMs,
|
|
19240
|
+
pendingBlockPublications.has(resource));
|
|
17591
19241
|
const retryAt = budget?.blockUntil > nowMs
|
|
17592
19242
|
? budget.blockUntil
|
|
17593
19243
|
: observer?.nextAt ?? decision.retryAt ?? wakes.controlAt;
|
|
@@ -17607,13 +19257,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17607
19257
|
states: Object.fromEntries(TAB_KEYS.map((key) => [key, {
|
|
17608
19258
|
unchangedCount: unchangedPolls[key],
|
|
17609
19259
|
inProgressCI: key === "actions" && actionsInProgress(),
|
|
19260
|
+
requested: requestedTabs.has(key),
|
|
17610
19261
|
}])),
|
|
17611
19262
|
});
|
|
17612
19263
|
scheduledWakePolls = planned.due;
|
|
17613
19264
|
pollDueAt = planned.dueAt;
|
|
17614
19265
|
backgroundIndex = planned.backgroundIndex;
|
|
17615
19266
|
const outcomes = await Promise.allSettled(
|
|
17616
|
-
planned.due.map(({ key, kind }) => requestTab(key, kind)),
|
|
19267
|
+
planned.due.map(({ key, kind }) => requestTab(key, kind, { detachTransport: true })),
|
|
17617
19268
|
);
|
|
17618
19269
|
const retryAt = governorControlRetryAt(Date.now(), runtime.refreshMs);
|
|
17619
19270
|
const retried = retryFailedPollBatch({ due: planned.due, outcomes,
|
|
@@ -17666,6 +19317,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17666
19317
|
|
|
17667
19318
|
async function runControlWake() {
|
|
17668
19319
|
if (cancelled) return;
|
|
19320
|
+
const controlAt = Date.now();
|
|
19321
|
+
const clockCause = wallClockRecovery(lastWallAt, controlAt, Math.min(runtime.refreshMs, 5_000));
|
|
19322
|
+
lastWallAt = controlAt;
|
|
19323
|
+
if (clockCause) {
|
|
19324
|
+
recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
|
|
19325
|
+
setNow(new Date(controlAt));
|
|
19326
|
+
}
|
|
17669
19327
|
for (const item of [...cleanupQueue]) retryPendingCleanup(item.cleanupKey, item);
|
|
17670
19328
|
await runtimeIdentityCoordinator.refresh();
|
|
17671
19329
|
if (cancelled) return;
|
|
@@ -17684,11 +19342,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17684
19342
|
}
|
|
17685
19343
|
const refreshed = currentScope
|
|
17686
19344
|
? await refreshRuntimeBudget(currentScope)
|
|
17687
|
-
: { ok: false,
|
|
19345
|
+
: { ok: false, ...(runtimeIdentityCoordinator?.inspect() ?? { reason: "stale" }) };
|
|
17688
19346
|
if (cancelled) return;
|
|
17689
19347
|
const checkedAt = Date.now();
|
|
17690
19348
|
const snapshot = currentScope ? inspectGovernor(currentScope, checkedAt) : refreshed;
|
|
17691
|
-
if (currentScope)
|
|
19349
|
+
if (currentScope) syncRuntimeQuotaProjection(currentScope, snapshot);
|
|
17692
19350
|
if (currentScope && snapshot.ok && pendingBlockPublications.size > 0) {
|
|
17693
19351
|
attemptPendingBlockPublications(currentScope, checkedAt, snapshot.value);
|
|
17694
19352
|
}
|
|
@@ -17750,14 +19408,27 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17750
19408
|
const livenessWake = createRecurringWake({
|
|
17751
19409
|
scheduler: wakeScheduler,
|
|
17752
19410
|
intervalMs: Math.min(runtime.refreshMs, 5000),
|
|
17753
|
-
run:
|
|
19411
|
+
run: () => {
|
|
17754
19412
|
if (cancelled || remoteSetupRef.current) return;
|
|
17755
19413
|
const at = Date.now();
|
|
19414
|
+
const clockCause = wallClockRecovery(lastWallAt, at, Math.min(runtime.refreshMs, 5_000));
|
|
19415
|
+
lastWallAt = at;
|
|
19416
|
+
for (const resource of RATE_RESOURCES) {
|
|
19417
|
+
if (["receipt-retry", "request-queue"].includes(recoveryRef.current[resource]?.code)) {
|
|
19418
|
+
clearRuntimeRecovery(resource);
|
|
19419
|
+
}
|
|
19420
|
+
}
|
|
19421
|
+
if (clockCause) {
|
|
19422
|
+
recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
|
|
19423
|
+
setNow(new Date(at));
|
|
19424
|
+
void controlWake();
|
|
19425
|
+
return;
|
|
19426
|
+
}
|
|
17756
19427
|
if (wakeScheduler.at("control") <= at ||
|
|
17757
|
-
!Number.isFinite(wakeScheduler.at("control")))
|
|
19428
|
+
!Number.isFinite(wakeScheduler.at("control"))) void controlWake();
|
|
17758
19429
|
if (pending.size > 0 || cleanupQueue.size > 0 ||
|
|
17759
19430
|
wakeScheduler.at("data") <= Date.now() ||
|
|
17760
|
-
Object.values(pollDueAt).some((due) => due <= Date.now()))
|
|
19431
|
+
Object.values(pollDueAt).some((due) => due <= Date.now())) void dataWake();
|
|
17761
19432
|
},
|
|
17762
19433
|
onError: (error) => {
|
|
17763
19434
|
if (!cancelled) pauseCoordination(TABS[activeIndexRef.current].key, error?.reason ?? "stale");
|
|
@@ -17773,7 +19444,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17773
19444
|
if (cancelled) return;
|
|
17774
19445
|
const currentScope = ensureScope(Date.now());
|
|
17775
19446
|
if (!currentScope) {
|
|
17776
|
-
|
|
19447
|
+
const identityFailure = runtimeIdentityCoordinator?.inspect();
|
|
19448
|
+
pauseCoordination(TABS[activeIndexRef.current].key,
|
|
19449
|
+
identityFailure?.reason ?? "unknown-scope", null, false, identityFailure?.retryAt);
|
|
17777
19450
|
armWake("control", governorControlRetryAt(Date.now(), runtime.refreshMs), controlWake);
|
|
17778
19451
|
return;
|
|
17779
19452
|
}
|
|
@@ -17783,7 +19456,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17783
19456
|
if (cancelled) return;
|
|
17784
19457
|
const checkedAt = Date.now();
|
|
17785
19458
|
const snapshot = inspectGovernor(currentScope, checkedAt);
|
|
17786
|
-
|
|
19459
|
+
syncRuntimeQuotaProjection(currentScope, snapshot);
|
|
17787
19460
|
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
17788
19461
|
if (snapshot.ok) {
|
|
17789
19462
|
controlEpochs = { ...snapshot.value.epochs };
|
|
@@ -17907,7 +19580,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17907
19580
|
const displayError = formatTabErrorForWidth(tabError, failureContext, Math.max(1, cols - 5));
|
|
17908
19581
|
const coordinationError = activeGovernorDecision?.coordinationError && !remoteSetup;
|
|
17909
19582
|
const coordinationReason = activeFailure ?? activeGovernorDecision?.reason ?? "unavailable";
|
|
17910
|
-
const
|
|
19583
|
+
const noticeReason = activeRecovery?.code ??
|
|
19584
|
+
(tabError?.kind === "fetch" ? tabError.verdict : null) ??
|
|
19585
|
+
(coordinationError || activeGovernorDecision?.mode === "paused" ? coordinationReason : null) ??
|
|
19586
|
+
(statusDisconnected ? runtime.connect === null ? "identity-unavailable"
|
|
19587
|
+
: activeGovernorDecision?.reason === "app-auth" ? "app-auth" : "disconnected" : null);
|
|
19588
|
+
// A continuous fault stays sustained when its explanation becomes more
|
|
19589
|
+
// specific. Only losing the fault (or switching tabs) restarts the debounce.
|
|
19590
|
+
const coordinationCondition = !remoteSetup && noticeReason ? tab.key : null;
|
|
17911
19591
|
useEffect(() => {
|
|
17912
19592
|
setVisibleCoordinationCondition(null);
|
|
17913
19593
|
if (coordinationCondition === null) return;
|
|
@@ -17919,10 +19599,21 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17919
19599
|
}, [coordinationCondition]);
|
|
17920
19600
|
const showCoordinationNotice = coordinationCondition !== null &&
|
|
17921
19601
|
visibleCoordinationCondition === coordinationCondition;
|
|
17922
|
-
const
|
|
17923
|
-
|
|
17924
|
-
|
|
17925
|
-
|
|
19602
|
+
const noticeResource = activeResource;
|
|
19603
|
+
const noticeSourceAt = collectorSourceAgeRef.current[tab.key]
|
|
19604
|
+
? now.getTime() - collectorSourceAgeRef.current[tab.key].ageMs -
|
|
19605
|
+
Math.max(0, performance.now() - collectorSourceAgeRef.current[tab.key].measuredAt)
|
|
19606
|
+
: lastOkRef.current[tab.key] ?? null;
|
|
19607
|
+
const noticeCause = activeRecovery ?? (noticeReason ? recoveryCause({
|
|
19608
|
+
reason: noticeReason === "other" ? "network-outage" : noticeReason,
|
|
19609
|
+
resource: noticeResource, sourceAt: noticeSourceAt,
|
|
19610
|
+
retryAt: activeGovernorDecision?.retryAt ?? activeGovernorDecision?.resetMs ??
|
|
19611
|
+
activeGovernorDecision?.notBefore ?? null,
|
|
19612
|
+
origin: tabError ? "source" : "coordination", tab: tabError ? tab.key : null,
|
|
19613
|
+
}) : null);
|
|
19614
|
+
const noticeLines = showCoordinationNotice && noticeCause
|
|
19615
|
+
? presentRecovery(noticeCause, { nowMs: now.getTime(), cols })
|
|
19616
|
+
: [!remoteSetup && displayError ? displayError : ""];
|
|
17926
19617
|
const noticeTone = showCoordinationNotice ? ATTENTION : displayError ? ERROR_TEXT : undefined;
|
|
17927
19618
|
const spin = SPINNER[frame % SPINNER.length];
|
|
17928
19619
|
|
|
@@ -17999,6 +19690,22 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17999
19690
|
});
|
|
18000
19691
|
}
|
|
18001
19692
|
|
|
19693
|
+
if (runtime.background === "on-demand" && tab.key !== "actions" && data[tab.key] !== null &&
|
|
19694
|
+
["watching", "shared", "stale"].includes(semanticStatus.kind) && !tabError &&
|
|
19695
|
+
!activeFailure && !(tab.key === "security" && securityBlind)) {
|
|
19696
|
+
semanticStatus = refreshStatus({ onDemand: true });
|
|
19697
|
+
}
|
|
19698
|
+
|
|
19699
|
+
// A settled on-demand view has no data wake to advance its visible age.
|
|
19700
|
+
// Tick only its display clock; source timestamps and acquisition stay idle.
|
|
19701
|
+
const showCachedAge = semanticStatus.kind === "cached" && staleFor !== null;
|
|
19702
|
+
useEffect(() => {
|
|
19703
|
+
if (!showCachedAge) return;
|
|
19704
|
+
setNow(new Date());
|
|
19705
|
+
const timer = setInterval(() => setNow(new Date()), 1000);
|
|
19706
|
+
return () => clearInterval(timer);
|
|
19707
|
+
}, [showCachedAge]);
|
|
19708
|
+
|
|
18002
19709
|
const allItems = items ?? [];
|
|
18003
19710
|
const tabOffsetRaw = offset[tab.key] ?? 0;
|
|
18004
19711
|
const selectedKey = selected[tab.key] ?? null;
|
|
@@ -18199,7 +19906,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
18199
19906
|
!showHelp &&
|
|
18200
19907
|
tooNarrow &&
|
|
18201
19908
|
e(Text, { dimColor: true, wrap: "truncate-end" }, "too narrow"),
|
|
18202
|
-
|
|
19909
|
+
...noticeLines.map((line, index) =>
|
|
19910
|
+
e(Text, { key: `recovery:${index}`, color: noticeTone, wrap: "truncate-end" }, line)),
|
|
18203
19911
|
!showHelp &&
|
|
18204
19912
|
!tooNarrow &&
|
|
18205
19913
|
!remoteSetup &&
|
|
@@ -18294,7 +20002,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
18294
20002
|
status: semanticStatus,
|
|
18295
20003
|
detail: activeGovernorDecision,
|
|
18296
20004
|
spin: showSpinner ? spin : null,
|
|
18297
|
-
stale:
|
|
20005
|
+
stale: semanticStatus.kind === "cached" && staleFor != null
|
|
20006
|
+
? formatDuration(Math.max(0, staleFor)) : staleLabel,
|
|
18298
20007
|
nowMs: now.getTime(),
|
|
18299
20008
|
interactive,
|
|
18300
20009
|
cols: frameCols,
|
|
@@ -18589,6 +20298,7 @@ export {
|
|
|
18589
20298
|
resolveEffectiveCredential,
|
|
18590
20299
|
identityRegistryRoot,
|
|
18591
20300
|
inspectIdentityRegistry,
|
|
20301
|
+
doctorCachedQuotaScope,
|
|
18592
20302
|
claimIdentityBootstrap,
|
|
18593
20303
|
finishIdentityBootstrap,
|
|
18594
20304
|
createIdentityCoordinator,
|
|
@@ -18633,6 +20343,10 @@ export {
|
|
|
18633
20343
|
budgetEpoch,
|
|
18634
20344
|
resourceReserve,
|
|
18635
20345
|
availableForGrant,
|
|
20346
|
+
governorChargedCost,
|
|
20347
|
+
governorAvailableForGrant,
|
|
20348
|
+
governorResourceDecision,
|
|
20349
|
+
governorResourceHeld,
|
|
18636
20350
|
nextExternalFactor,
|
|
18637
20351
|
resourceDecision,
|
|
18638
20352
|
governorPhaseOffset,
|
|
@@ -18658,6 +20372,7 @@ export {
|
|
|
18658
20372
|
serializeGovernorState,
|
|
18659
20373
|
readGovernorState,
|
|
18660
20374
|
writeGovernorState,
|
|
20375
|
+
writeGovernorQuotaState,
|
|
18661
20376
|
pidIsDead,
|
|
18662
20377
|
claimGovernorLock,
|
|
18663
20378
|
releaseGovernorLock,
|
|
@@ -18685,6 +20400,9 @@ export {
|
|
|
18685
20400
|
readIntentDecision,
|
|
18686
20401
|
cancelIntent,
|
|
18687
20402
|
startReservation,
|
|
20403
|
+
issueGovernorDispatch,
|
|
20404
|
+
terminalGovernorDispatch,
|
|
20405
|
+
acknowledgeSealedGeneration,
|
|
18688
20406
|
completeReservation,
|
|
18689
20407
|
settleReservationWithBudgetObservations,
|
|
18690
20408
|
recordResourceBlock,
|
|
@@ -18846,7 +20564,9 @@ export {
|
|
|
18846
20564
|
acquisitionRequestMetrics,
|
|
18847
20565
|
acquisitionFailureHold,
|
|
18848
20566
|
setRuntimeAcquisitionHold,
|
|
20567
|
+
clearScheduledRuntimeControlHold,
|
|
18849
20568
|
loadAcquisitionStore,
|
|
20569
|
+
readAcquisitionSnapshot,
|
|
18850
20570
|
createAcquisitionEngine,
|
|
18851
20571
|
pollPolicyInterval,
|
|
18852
20572
|
advanceUnchangedCount,
|
|
@@ -18882,11 +20602,27 @@ export {
|
|
|
18882
20602
|
runtimeIntentGate,
|
|
18883
20603
|
rateLimitBlockDecision,
|
|
18884
20604
|
coordinationNotice,
|
|
20605
|
+
recoveryCause,
|
|
20606
|
+
shouldClearRecoveryCause,
|
|
20607
|
+
recoveryRetryAt,
|
|
20608
|
+
terminalizationRecoveryCode,
|
|
20609
|
+
terminalizationBacklogForResource,
|
|
20610
|
+
receiptCapabilityFromReservation,
|
|
20611
|
+
requestFailureRecoveryCode,
|
|
20612
|
+
requestFailureRecovery,
|
|
20613
|
+
wallClockRecovery,
|
|
20614
|
+
collectorSourceAdvanced,
|
|
20615
|
+
cleanupQueueForResource,
|
|
20616
|
+
presentRecovery,
|
|
20617
|
+
recordRecoveryTransition,
|
|
20618
|
+
safeRecordRecoveryTransition,
|
|
20619
|
+
readRecoveryTransitions,
|
|
18885
20620
|
mergeRateLimitBlockPublications,
|
|
18886
20621
|
hydrateRateLimitBlockPublication,
|
|
18887
20622
|
retryRateLimitBlockPublication,
|
|
18888
20623
|
rateLimitBlockProbeRecovered,
|
|
18889
20624
|
admitGovernorOperation,
|
|
20625
|
+
queueTerminalization,
|
|
18890
20626
|
runAdmittedOperation,
|
|
18891
20627
|
abortableDelay,
|
|
18892
20628
|
awaitCollectorReservation,
|
|
@@ -18912,6 +20648,7 @@ export {
|
|
|
18912
20648
|
demandedPageCount,
|
|
18913
20649
|
mergeDemandedPages,
|
|
18914
20650
|
operationPausedUntil,
|
|
20651
|
+
pauseOperation,
|
|
18915
20652
|
LIST_LIMIT,
|
|
18916
20653
|
readGraphqlObserver,
|
|
18917
20654
|
GRAPHQL_QUERIES,
|