gh-glance 0.15.2 → 0.16.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +69 -1
- package/README.md +76 -25
- package/index.mjs +2257 -501
- package/package.json +1 -1
package/index.mjs
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
// disarmDevBuildLeak() for how that path is kept survivable rather than fatal.
|
|
23
23
|
process.env.NODE_ENV ??= "production";
|
|
24
24
|
|
|
25
|
-
import { execFile, spawn } from "node:child_process";
|
|
25
|
+
import { execFile, execFileSync, spawn } from "node:child_process";
|
|
26
26
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
27
27
|
import { createHash, createHmac, createPrivateKey, randomUUID, sign as cryptoSign, timingSafeEqual } from "node:crypto";
|
|
28
28
|
import { EventEmitter } from "node:events";
|
|
@@ -207,7 +207,7 @@ const STALE_AFTER_MS = 30_000;
|
|
|
207
207
|
// A coordination hold must persist long enough to be meaningful before it
|
|
208
208
|
// takes the notice row. Short startup and lock-contention states recover on
|
|
209
209
|
// their own and should not present themselves as user-visible failures.
|
|
210
|
-
const COORDINATION_NOTICE_AFTER_MS =
|
|
210
|
+
const COORDINATION_NOTICE_AFTER_MS = 800;
|
|
211
211
|
|
|
212
212
|
// Mutable because the fetchers below are defined before argv is parsed, and the
|
|
213
213
|
// argv block is what fills this in. Everything here has a working default, so
|
|
@@ -221,7 +221,7 @@ const runtime = {
|
|
|
221
221
|
host: null,
|
|
222
222
|
repoExplicit: false,
|
|
223
223
|
refreshMs: REFRESH_MS,
|
|
224
|
-
background: "
|
|
224
|
+
background: "on-demand",
|
|
225
225
|
verbose: false,
|
|
226
226
|
connect: null,
|
|
227
227
|
initialTabIndex: 0,
|
|
@@ -734,7 +734,9 @@ const pausedOperations = new Map();
|
|
|
734
734
|
|
|
735
735
|
function pauseOperation(operation, untilMs) {
|
|
736
736
|
if (!Number.isFinite(untilMs)) return;
|
|
737
|
-
|
|
737
|
+
const until = Math.max(pausedOperations.get(operation) ?? 0, untilMs);
|
|
738
|
+
pausedOperations.set(operation, until);
|
|
739
|
+
return () => { if (pausedOperations.get(operation) === until) pausedOperations.delete(operation); };
|
|
738
740
|
}
|
|
739
741
|
|
|
740
742
|
function operationPausedUntil(operation, now = Date.now()) {
|
|
@@ -744,7 +746,8 @@ function operationPausedUntil(operation, now = Date.now()) {
|
|
|
744
746
|
return until;
|
|
745
747
|
}
|
|
746
748
|
|
|
747
|
-
async function runGh(args, { signal, operation, input = null, execute = null
|
|
749
|
+
async function runGh(args, { signal, operation, input = null, execute = null,
|
|
750
|
+
closeWaitMs = 15_000 } = {}) {
|
|
748
751
|
if (operationCost(operation) === null) {
|
|
749
752
|
throw new Error(`undeclared gh operation: ${operation ?? "missing"}`);
|
|
750
753
|
}
|
|
@@ -756,11 +759,32 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
756
759
|
const requestNow = typeof bound?.now === "function" ? bound.now : Date.now;
|
|
757
760
|
const permit = identityCoordinator && !local
|
|
758
761
|
? await acquireIdentityHttpPermit(identityCoordinator, { signal, now: requestNow,
|
|
762
|
+
operationDeadline: CONTROL_OPERATIONS.includes(operation) ? undefined : bound?.receiptCapability?.deadline,
|
|
759
763
|
...(typeof bound?.httpWait === "function" ? { wait: bound.httpWait } : {}) }) : null;
|
|
760
764
|
const startedAt = requestNow();
|
|
761
765
|
let requestError = null;
|
|
762
766
|
let requestStdout = null;
|
|
763
767
|
let requestStarted = false;
|
|
768
|
+
let knownNeverStarted = false;
|
|
769
|
+
let requestSettled = false;
|
|
770
|
+
let successfulPromise = false;
|
|
771
|
+
let dispatchEvidence = null;
|
|
772
|
+
let dispatchCapability = null;
|
|
773
|
+
let childCloseProof = null;
|
|
774
|
+
let activeDispatchKey = null;
|
|
775
|
+
let activeDispatchReleased = false;
|
|
776
|
+
const releaseActiveDispatch = () => {
|
|
777
|
+
if (!activeDispatchKey || activeDispatchReleased) return;
|
|
778
|
+
activeDispatchReleased = true;
|
|
779
|
+
const remaining = (activeOwnerDispatches.get(activeDispatchKey) ?? 1) - 1;
|
|
780
|
+
if (remaining > 0) activeOwnerDispatches.set(activeDispatchKey, remaining);
|
|
781
|
+
else activeOwnerDispatches.delete(activeDispatchKey);
|
|
782
|
+
if (dispatchCapability && !dispatchCapability.controlResource) {
|
|
783
|
+
const sealed = inspectGovernor({ ...bound, identityProvider: null }, requestNow());
|
|
784
|
+
const nonce = sealed.ok && sealed.value.debtGroups[dispatchCapability.generation]?.nonce;
|
|
785
|
+
if (nonce) acknowledgeSealedGeneration(bound, dispatchCapability, nonce, requestNow());
|
|
786
|
+
}
|
|
787
|
+
};
|
|
764
788
|
try {
|
|
765
789
|
assertBoundCredential(bound);
|
|
766
790
|
if (identityCoordinator && operation === "budget-core-observer") {
|
|
@@ -789,6 +813,23 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
789
813
|
}
|
|
790
814
|
}
|
|
791
815
|
assertBoundCredential(bound);
|
|
816
|
+
if (!local && RATE_RESOURCES.some((resource) => operationCost(operation)[resource] > 0)) {
|
|
817
|
+
dispatchCapability = operation === "budget-core-observer"
|
|
818
|
+
? control?.value?.receiptCapability
|
|
819
|
+
: operation === "graphql-observer"
|
|
820
|
+
? bound?.controlReceiptCapability : bound?.receiptCapability;
|
|
821
|
+
if (dispatchCapability) {
|
|
822
|
+
activeDispatchKey = ownerDispatchKey(bound, dispatchCapability);
|
|
823
|
+
activeOwnerDispatches.set(activeDispatchKey,
|
|
824
|
+
(activeOwnerDispatches.get(activeDispatchKey) ?? 0) + 1);
|
|
825
|
+
}
|
|
826
|
+
const issued = issueGovernorDispatch(bound, dispatchCapability, operation, requestNow());
|
|
827
|
+
if (!issued.ok) {
|
|
828
|
+
releaseActiveDispatch();
|
|
829
|
+
throw new Error(`API receipt paused (${issued.reason})`);
|
|
830
|
+
}
|
|
831
|
+
dispatchEvidence = issued.value;
|
|
832
|
+
}
|
|
792
833
|
const executeChild = execute ?? bound?.executeGh ?? execFileAsync;
|
|
793
834
|
const pending = executeChild("gh", args, {
|
|
794
835
|
timeout: GH_TIMEOUT_MS,
|
|
@@ -798,6 +839,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
798
839
|
signal,
|
|
799
840
|
});
|
|
800
841
|
requestStarted = true;
|
|
842
|
+
if (Number.isSafeInteger(pending.child?.pid) &&
|
|
843
|
+
typeof pending.child.once === "function") {
|
|
844
|
+
childCloseProof = new Promise((resolve) => pending.child.once("close", resolve));
|
|
845
|
+
}
|
|
846
|
+
if (dispatchEvidence && Number.isSafeInteger(pending.child?.pid)) {
|
|
847
|
+
markGovernorDispatchChild(bound, dispatchCapability,
|
|
848
|
+
dispatchEvidence.sequence, pending.child.pid, requestNow());
|
|
849
|
+
}
|
|
801
850
|
if (typeof input === "string") {
|
|
802
851
|
// A child killed by the timeout or the abort signal closes stdin under
|
|
803
852
|
// us; that EPIPE is the kill's consequence, not the failure worth
|
|
@@ -805,16 +854,73 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
805
854
|
pending.child.stdin?.on("error", () => {});
|
|
806
855
|
pending.child.stdin?.end(input);
|
|
807
856
|
}
|
|
808
|
-
|
|
857
|
+
let stdout;
|
|
858
|
+
try {
|
|
859
|
+
({ stdout } = await pending);
|
|
860
|
+
requestSettled = true;
|
|
861
|
+
successfulPromise = true;
|
|
862
|
+
} catch (error) {
|
|
863
|
+
requestSettled = true;
|
|
864
|
+
throw error;
|
|
865
|
+
}
|
|
809
866
|
requestStdout = stdout;
|
|
810
|
-
logGh(args, startedAt, `ok ${stdout.length}B`);
|
|
811
|
-
return stdout;
|
|
812
867
|
} catch (err) {
|
|
813
868
|
requestError = err;
|
|
869
|
+
if (!requestStarted && !dispatchEvidence) err.notStarted ??= true;
|
|
870
|
+
knownNeverStarted = !requestStarted && err?.notStarted === true;
|
|
814
871
|
if (requestStarted && isRecord(err)) err.httpStarted = true;
|
|
815
872
|
logGh(args, startedAt, `FAILED ${shortErr(err)}`);
|
|
816
873
|
throw err;
|
|
817
874
|
} finally {
|
|
875
|
+
let childClosed = !requestStarted;
|
|
876
|
+
if (requestSettled && childCloseProof) {
|
|
877
|
+
let timer;
|
|
878
|
+
childClosed = await Promise.race([
|
|
879
|
+
childCloseProof.then(() => true),
|
|
880
|
+
new Promise((resolve) => { timer = setTimeout(() => resolve(false), closeWaitMs); }),
|
|
881
|
+
]);
|
|
882
|
+
clearTimeout(timer);
|
|
883
|
+
}
|
|
884
|
+
if (requestSettled && !childClosed) {
|
|
885
|
+
requestError ??= Object.assign(new Error("GitHub child close unverified"), { httpStarted: true });
|
|
886
|
+
}
|
|
887
|
+
if (successfulPromise) {
|
|
888
|
+
logGh(args, startedAt, childClosed
|
|
889
|
+
? `ok ${requestStdout.length}B` : "FAILED GitHub child close unverified");
|
|
890
|
+
}
|
|
891
|
+
if (dispatchEvidence && (requestSettled && childClosed || knownNeverStarted)) {
|
|
892
|
+
const finishDispatch = () => {
|
|
893
|
+
const result = terminalGovernorDispatch(bound,
|
|
894
|
+
dispatchCapability, dispatchEvidence.sequence, requestNow(),
|
|
895
|
+
{ neverStarted: knownNeverStarted });
|
|
896
|
+
if (result.ok || result.reason === "compacted") releaseActiveDispatch();
|
|
897
|
+
return result;
|
|
898
|
+
};
|
|
899
|
+
const terminal = await retryIdentityCompletion(finishDispatch, {
|
|
900
|
+
timeoutMs: 15_000, shouldRetry: () => true,
|
|
901
|
+
});
|
|
902
|
+
if (!terminal.ok && identityCoordinator?.deferCompletion) {
|
|
903
|
+
identityCoordinator.deferCompletion(
|
|
904
|
+
`dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
|
|
905
|
+
finishDispatch,
|
|
906
|
+
);
|
|
907
|
+
}
|
|
908
|
+
} else if (dispatchEvidence && requestSettled && childCloseProof) {
|
|
909
|
+
// A promise can settle before its subprocess closes. Keep the issued
|
|
910
|
+
// charge until close proves that no more HTTP can escape this child.
|
|
911
|
+
void childCloseProof.then(async () => {
|
|
912
|
+
const finishDispatch = () => terminalGovernorDispatch(bound,
|
|
913
|
+
dispatchCapability, dispatchEvidence.sequence, requestNow());
|
|
914
|
+
const terminal = await retryIdentityCompletion(finishDispatch, {
|
|
915
|
+
timeoutMs: 15_000, shouldRetry: () => true,
|
|
916
|
+
});
|
|
917
|
+
if (terminal.ok || terminal.reason === "compacted") releaseActiveDispatch();
|
|
918
|
+
else identityCoordinator?.deferCompletion?.(
|
|
919
|
+
`dispatch:${dispatchCapability.reservationId}:${dispatchEvidence.sequence}`,
|
|
920
|
+
finishDispatch,
|
|
921
|
+
);
|
|
922
|
+
});
|
|
923
|
+
}
|
|
818
924
|
if (permit) {
|
|
819
925
|
const release = () => releaseIdentityHttpPermit(identityCoordinator, permit, requestError, requestNow());
|
|
820
926
|
const released = await retryIdentityCompletion(release, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
|
|
@@ -822,11 +928,14 @@ async function runGh(args, { signal, operation, input = null, execute = null } =
|
|
|
822
928
|
}
|
|
823
929
|
if (control?.ok) {
|
|
824
930
|
const settle = () => settleIdentityControl(identityCoordinator, control.value,
|
|
825
|
-
requestStdout ?? requestError?.stdout, requestNow()
|
|
931
|
+
childClosed ? requestStdout ?? requestError?.stdout : null, requestNow(),
|
|
932
|
+
{ neverStarted: !requestStarted && (!dispatchEvidence || knownNeverStarted) });
|
|
826
933
|
const settled = await retryIdentityCompletion(settle, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !identityCoordinator.isClosed() });
|
|
827
934
|
if (!settled.ok) identityCoordinator.deferCompletion(`control:${control.value.id}`, settle);
|
|
828
935
|
}
|
|
829
936
|
}
|
|
937
|
+
if (successfulPromise && requestError) throw requestError;
|
|
938
|
+
return requestStdout;
|
|
830
939
|
}
|
|
831
940
|
|
|
832
941
|
function parseGhApiResponse(stdout) {
|
|
@@ -1638,7 +1747,6 @@ async function fetchActions(signal, {
|
|
|
1638
1747
|
}
|
|
1639
1748
|
const responses = [runsResponse];
|
|
1640
1749
|
let requestMetrics = runsResponse.requestMetrics ?? restResponseRequestMetrics(runsResponse);
|
|
1641
|
-
const uncertainReceipts = [];
|
|
1642
1750
|
const runs = parseActionsRuns(runsResponse.body);
|
|
1643
1751
|
const startedAt = Date.now();
|
|
1644
1752
|
let nextCatalog = catalog;
|
|
@@ -1662,7 +1770,6 @@ async function fetchActions(signal, {
|
|
|
1662
1770
|
request,
|
|
1663
1771
|
}),
|
|
1664
1772
|
});
|
|
1665
|
-
uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
|
|
1666
1773
|
if (admitted.ok && admitted.value) {
|
|
1667
1774
|
responses.push(admitted.value);
|
|
1668
1775
|
requestMetrics = mergeAcquisitionRequestMetrics(
|
|
@@ -1695,7 +1802,6 @@ async function fetchActions(signal, {
|
|
|
1695
1802
|
failedRequests: requestMetrics.failedRequests ?? 0,
|
|
1696
1803
|
coreUnitsTotal: requestMetrics.coreUnits ?? 0,
|
|
1697
1804
|
requestMetrics,
|
|
1698
|
-
uncertainReceipts,
|
|
1699
1805
|
graphqlSpent: GRAPHQL_PER_FETCH.actions,
|
|
1700
1806
|
stagedEntities: batch.stagedEntities,
|
|
1701
1807
|
observations: batch.observations,
|
|
@@ -1896,7 +2002,6 @@ async function fetchGraphqlList(kind, mapRow, {
|
|
|
1896
2002
|
throw error;
|
|
1897
2003
|
}
|
|
1898
2004
|
const observations = [...first.observations];
|
|
1899
|
-
const uncertainReceipts = [];
|
|
1900
2005
|
const repositoryIdentity = normalizeRepositoryIdentity(first.data?.repository);
|
|
1901
2006
|
// Only the first page. Pages past it opened their own reservation inside
|
|
1902
2007
|
// runAdmittedOperation and settle against it, so adding them here charges the
|
|
@@ -1934,7 +2039,6 @@ async function fetchGraphqlList(kind, mapRow, {
|
|
|
1934
2039
|
waitMs: GOVERNOR_ADMISSION_WAIT_MS,
|
|
1935
2040
|
run: (admittedSignal) => fetchPage(kind, { signal: admittedSignal, after: cursor, operation: `page:${kind}` }),
|
|
1936
2041
|
});
|
|
1937
|
-
uncertainReceipts.push(...(admitted.uncertainReceipts ?? []));
|
|
1938
2042
|
// A later page that is denied, fails, or returns errors leaves the rows
|
|
1939
2043
|
// already gathered exactly as they are. Losing page one because page two
|
|
1940
2044
|
// was refused would turn a budget decision into data loss.
|
|
@@ -1988,7 +2092,6 @@ async function fetchGraphqlList(kind, mapRow, {
|
|
|
1988
2092
|
httpRequests: requestMetrics.httpRequests ?? 0,
|
|
1989
2093
|
failedRequests: requestMetrics.failedRequests ?? 0,
|
|
1990
2094
|
requestMetrics,
|
|
1991
|
-
uncertainReceipts,
|
|
1992
2095
|
observations,
|
|
1993
2096
|
incomplete,
|
|
1994
2097
|
repositoryIdentity,
|
|
@@ -2186,6 +2289,12 @@ function sharedLaneEvidence(value) {
|
|
|
2186
2289
|
// Below this many completed shared calls in a probe window, external-spend
|
|
2187
2290
|
// inference is noise. Under the threshold the loop retains the previous factor.
|
|
2188
2291
|
const MIN_SAMPLE_CALLS = 5;
|
|
2292
|
+
// Pacing divides the spendable share by the external factor, so a large factor
|
|
2293
|
+
// compounds: gh-glance's own paid calls shrink, the next sample's ratio grows,
|
|
2294
|
+
// and the lane is pushed past the reset while quota sits unused. The reserve,
|
|
2295
|
+
// not this divisor, is what keeps gh-glance off the floor, so the divisor can
|
|
2296
|
+
// be bounded without letting external load exhaust the account.
|
|
2297
|
+
const MAX_PACING_EXTERNAL_FACTOR = 4;
|
|
2189
2298
|
|
|
2190
2299
|
const REQUEST_PRIORITIES = Object.freeze({
|
|
2191
2300
|
manual: 0,
|
|
@@ -2265,6 +2374,41 @@ function reservationCost(reservation, resource, leases, nowMs) {
|
|
|
2265
2374
|
return Number.isFinite(cost) && cost > 0 ? Math.max(0, cost - accounted) : 0;
|
|
2266
2375
|
}
|
|
2267
2376
|
|
|
2377
|
+
function governorAccountingSafe(state, nowMs) {
|
|
2378
|
+
return RATE_RESOURCES.every((resource) =>
|
|
2379
|
+
Number.isSafeInteger(governorChargedCost(state, resource, nowMs)));
|
|
2380
|
+
}
|
|
2381
|
+
|
|
2382
|
+
function governorChargedCost(state, resource, nowMs) {
|
|
2383
|
+
if (!state || !RATE_RESOURCES.includes(resource)) return Number.NaN;
|
|
2384
|
+
let total = state.debt?.[resource]?.unresolvedUnits +
|
|
2385
|
+
state.debt?.[resource]?.quiescentUnits;
|
|
2386
|
+
if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
|
|
2387
|
+
for (const reservation of [state.controlReceipts?.[resource],
|
|
2388
|
+
...Object.values(state.reservations ?? {})]) {
|
|
2389
|
+
total += reservationCost(reservation, resource, state.leases, nowMs);
|
|
2390
|
+
if (!Number.isSafeInteger(total) || total < 0) return Number.NaN;
|
|
2391
|
+
}
|
|
2392
|
+
return total;
|
|
2393
|
+
}
|
|
2394
|
+
|
|
2395
|
+
function governorAvailableForGrant(state, resource, nowMs) {
|
|
2396
|
+
return availableForGrant({ budget: state?.budgets?.[resource], resource, nowMs,
|
|
2397
|
+
chargedCost: governorChargedCost(state, resource, nowMs) });
|
|
2398
|
+
}
|
|
2399
|
+
|
|
2400
|
+
function governorResourceDecision(state, resource, nowMs, cost) {
|
|
2401
|
+
return resourceDecision({ budget: state?.budgets?.[resource], resource, nowMs, cost,
|
|
2402
|
+
chargedCost: governorChargedCost(state, resource, nowMs) });
|
|
2403
|
+
}
|
|
2404
|
+
|
|
2405
|
+
function governorResourceHeld(state, resource, nowMs, pendingBlock = false) {
|
|
2406
|
+
const budget = state?.budgets?.[resource];
|
|
2407
|
+
const decision = governorAvailableForGrant(state, resource, nowMs);
|
|
2408
|
+
return pendingBlock || budget?.blockUntil > nowMs ||
|
|
2409
|
+
decision.mode === "paused" || decision.mode === "open" && decision.spendable <= 0;
|
|
2410
|
+
}
|
|
2411
|
+
|
|
2268
2412
|
function availableForGrant({
|
|
2269
2413
|
budget,
|
|
2270
2414
|
resource = budget?.resource,
|
|
@@ -2379,7 +2523,8 @@ function resourceDecision({
|
|
|
2379
2523
|
epoch: capacity.epoch,
|
|
2380
2524
|
};
|
|
2381
2525
|
}
|
|
2382
|
-
const callsPerMs = capacity.spendable / (capacity.resetMs - nowMs) /
|
|
2526
|
+
const callsPerMs = capacity.spendable / (capacity.resetMs - nowMs) /
|
|
2527
|
+
Math.min(externalFactor, MAX_PACING_EXTERNAL_FACTOR);
|
|
2383
2528
|
if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) {
|
|
2384
2529
|
return {
|
|
2385
2530
|
mode: "paused",
|
|
@@ -2517,6 +2662,7 @@ function scheduleIntents({
|
|
|
2517
2662
|
leases = {},
|
|
2518
2663
|
budgets = {},
|
|
2519
2664
|
reservations = [],
|
|
2665
|
+
debtCosts = { core: 0, graphql: 0 },
|
|
2520
2666
|
lanes = {},
|
|
2521
2667
|
cursors = {},
|
|
2522
2668
|
nowMs,
|
|
@@ -2551,7 +2697,7 @@ function scheduleIntents({
|
|
|
2551
2697
|
|
|
2552
2698
|
const chargedTotals = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
2553
2699
|
resource,
|
|
2554
|
-
reservations.reduce(
|
|
2700
|
+
debtCosts[resource] + reservations.reduce(
|
|
2555
2701
|
(total, reservation) => total + reservationCost(reservation, resource, leases, nowMs),
|
|
2556
2702
|
0,
|
|
2557
2703
|
),
|
|
@@ -2694,17 +2840,17 @@ function scheduleIntents({
|
|
|
2694
2840
|
}
|
|
2695
2841
|
|
|
2696
2842
|
// ---------- Credential identities and restart-safe coordination ----------
|
|
2697
|
-
const IDENTITY_VERSION =
|
|
2843
|
+
const IDENTITY_VERSION = 2;
|
|
2698
2844
|
const IDENTITY_ATTEMPT_WINDOW_MS = 15 * 60_000;
|
|
2699
2845
|
const IDENTITY_ATTEMPT_LIMIT = 3;
|
|
2700
2846
|
const IDENTITY_HOST_ATTEMPT_LIMIT = 12;
|
|
2701
2847
|
const IDENTITY_MAX_ATTEMPTS = 4096;
|
|
2702
2848
|
const HTTP_START_GAP_MS = 250;
|
|
2703
2849
|
const HTTP_MAX_WAITERS = 128;
|
|
2704
|
-
|
|
2705
|
-
//
|
|
2706
|
-
//
|
|
2707
|
-
//
|
|
2850
|
+
const HTTP_MAX_PERMITS = 3;
|
|
2851
|
+
// Retain the existing bounded permit lease for managed requests. This bounds
|
|
2852
|
+
// unexpired admissions, not surviving OS children after suspension or owner
|
|
2853
|
+
// death; quota receipts separately retain uncertain transport charges.
|
|
2708
2854
|
const HTTP_PERMIT_MAX_MS = GH_TIMEOUT_MS + 5_000;
|
|
2709
2855
|
// The longest a primary window can run. An attempt that never obtained reset
|
|
2710
2856
|
// evidence is still covered once a full window has demonstrably elapsed since it
|
|
@@ -2738,11 +2884,15 @@ function retryableCoordination(reason) {
|
|
|
2738
2884
|
return RETRYABLE_COORDINATION_REASONS.includes(reason);
|
|
2739
2885
|
}
|
|
2740
2886
|
|
|
2741
|
-
//
|
|
2742
|
-
//
|
|
2743
|
-
// test a stopped process, or one whose release never won the lock, holds the
|
|
2744
|
-
// single machine-wide permit forever and every pane queues behind it.
|
|
2887
|
+
// Managed permit leases retain their existing age/dead-owner reclamation law.
|
|
2888
|
+
// Reclaiming a slot never retires its quota receipt or proves child quiescence.
|
|
2745
2889
|
function staleHttpPermit(permit, now, kill = process.kill.bind(process)) {
|
|
2890
|
+
// A dead v1 parent does not prove its untracked child ended. Reserve its
|
|
2891
|
+
// one slot until a subsequent verified boot; two managed slots remain usable.
|
|
2892
|
+
if (permit.kind === "legacy") {
|
|
2893
|
+
const bootId = coordinationBootId();
|
|
2894
|
+
return permit.bootId !== null && bootId !== null && permit.bootId !== bootId;
|
|
2895
|
+
}
|
|
2746
2896
|
if (pidIsDead(permit.pid, kill)) return true;
|
|
2747
2897
|
return Number.isFinite(permit.startedAt) && now - permit.startedAt > HTTP_PERMIT_MAX_MS;
|
|
2748
2898
|
}
|
|
@@ -2813,8 +2963,8 @@ function emptyIdentityRegistry() {
|
|
|
2813
2963
|
return { version: IDENTITY_VERSION, migration: { activated: false, holdUntil: 0 }, identities: {}, attempts: {}, hosts: {} };
|
|
2814
2964
|
}
|
|
2815
2965
|
|
|
2816
|
-
function normalizeIdentityRegistry(raw) {
|
|
2817
|
-
if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !==
|
|
2966
|
+
function normalizeIdentityRegistry(raw, version = IDENTITY_VERSION) {
|
|
2967
|
+
if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !== version ||
|
|
2818
2968
|
!exactKeys(raw.migration, ["activated", "holdUntil"]) || typeof raw.migration.activated !== "boolean" ||
|
|
2819
2969
|
!Number.isFinite(raw.migration.holdUntil) || raw.migration.holdUntil < 0 ||
|
|
2820
2970
|
!isRecord(raw.identities) || !isRecord(raw.attempts) || !isRecord(raw.hosts) ||
|
|
@@ -2837,13 +2987,20 @@ function normalizeIdentityRegistry(raw) {
|
|
|
2837
2987
|
}
|
|
2838
2988
|
for (const [host, transport] of Object.entries(raw.hosts)) {
|
|
2839
2989
|
if (!isRecord(transport)) return null;
|
|
2840
|
-
const keys = ["lastStartedAt", "cooldownUntil", "permit"];
|
|
2990
|
+
const keys = ["lastStartedAt", "cooldownUntil", version === 1 ? "permit" : "permits"];
|
|
2841
2991
|
if (Object.hasOwn(transport, "waiters")) keys.push("waiters");
|
|
2842
2992
|
if (Object.hasOwn(transport, "throttle")) keys.push("throttle");
|
|
2843
2993
|
if (!normalizeHost(host) || !exactKeys(transport, keys) ||
|
|
2844
2994
|
!time(transport.lastStartedAt) || !time(transport.cooldownUntil)) return null;
|
|
2845
|
-
|
|
2846
|
-
|
|
2995
|
+
const permits = version === 1 ? (transport.permit === null ? [] : [transport.permit]) : transport.permits;
|
|
2996
|
+
if (!Array.isArray(permits) || permits.length > HTTP_MAX_PERMITS ||
|
|
2997
|
+
new Set(permits.map((permit) => permit?.nonce)).size !== permits.length ||
|
|
2998
|
+
permits.filter((permit) => permit?.kind === "legacy").length > 1 ||
|
|
2999
|
+
permits.some((permit) => !exactKeys(permit, version === 1
|
|
3000
|
+
? ["pid", "nonce", "startedAt"] : ["pid", "nonce", "startedAt", "kind", "bootId"]) ||
|
|
3001
|
+
!Number.isSafeInteger(permit.pid) || permit.pid < 1 || !validGovernorId(permit.nonce) || !time(permit.startedAt) ||
|
|
3002
|
+
version !== 1 && (!["request", "bootstrap", "legacy"].includes(permit.kind) ||
|
|
3003
|
+
permit.bootId !== null && (typeof permit.bootId !== "string" || permit.bootId.length > 128)))) return null;
|
|
2847
3004
|
if (!Object.hasOwn(transport, "throttle")) transport.throttle = emptyTransportThrottle();
|
|
2848
3005
|
if (!isRecord(transport.throttle) || !exactKeys(transport.throttle, ["attempts", "lastAt", "paused"]) ||
|
|
2849
3006
|
!Number.isSafeInteger(transport.throttle.attempts) || transport.throttle.attempts < 0 ||
|
|
@@ -2874,8 +3031,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
|
|
|
2874
3031
|
let state;
|
|
2875
3032
|
if (raw === null) state = emptyIdentityRegistry();
|
|
2876
3033
|
else {
|
|
2877
|
-
try {
|
|
2878
|
-
|
|
3034
|
+
try {
|
|
3035
|
+
const parsed = JSON.parse(raw);
|
|
3036
|
+
state = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
|
|
3037
|
+
} catch { return { ok: false, reason: "corrupt" }; }
|
|
2879
3038
|
}
|
|
2880
3039
|
if (!state) return { ok: false, reason: "corrupt" };
|
|
2881
3040
|
// Captured before the prunes below so their deletions still count as a
|
|
@@ -2883,6 +3042,19 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
|
|
|
2883
3042
|
// funnels through here, and each one used to serialize and rename the
|
|
2884
3043
|
// registry while holding the one machine-wide lock.
|
|
2885
3044
|
const before = JSON.stringify(state);
|
|
3045
|
+
if (state.version === 1) {
|
|
3046
|
+
// Keep v1 writable until a live owner commits its response and cooldown.
|
|
3047
|
+
// Returning the shared transient reason preserves cached identity/leases.
|
|
3048
|
+
if (Object.values(state.hosts).some((transport) =>
|
|
3049
|
+
transport.permit && !pidIsDead(transport.permit.pid, kill))) return { ok: false, reason: "busy" };
|
|
3050
|
+
for (const transport of Object.values(state.hosts)) {
|
|
3051
|
+
transport.permits = transport.permit
|
|
3052
|
+
? [{ ...transport.permit, kind: "legacy", bootId: coordinationBootId() }] : [];
|
|
3053
|
+
delete transport.permit;
|
|
3054
|
+
}
|
|
3055
|
+
state.version = IDENTITY_VERSION;
|
|
3056
|
+
}
|
|
3057
|
+
|
|
2886
3058
|
for (const transport of Object.values(state.hosts)) {
|
|
2887
3059
|
transport.waiters = transport.waiters.filter((waiter) => waiter.deadline > now && !pidIsDead(waiter.pid, kill));
|
|
2888
3060
|
}
|
|
@@ -2913,9 +3085,10 @@ function withIdentityRegistry(root, operation, { now = Date.now(), kill = proces
|
|
|
2913
3085
|
// stops the entry from being retained for the life of the install.
|
|
2914
3086
|
: budget?.resetMs > attempt.startedAt + IDENTITY_UNCERTAIN_MAX_MS;
|
|
2915
3087
|
if (!attempt.accounted && !resetCovered) return { ok: true, value: false };
|
|
2916
|
-
|
|
2917
|
-
|
|
2918
|
-
|
|
3088
|
+
// Retiring registry attempt history never retires quota debt. A reset
|
|
3089
|
+
// proves a counter window changed, not that a suspended child cannot
|
|
3090
|
+
// dispatch after it resumes.
|
|
3091
|
+
return { ok: true, value: true };
|
|
2919
3092
|
});
|
|
2920
3093
|
if (!retired.ok) return retired;
|
|
2921
3094
|
if (retired.value) delete state.attempts[id];
|
|
@@ -3010,9 +3183,10 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
|
|
|
3010
3183
|
return withIdentityRegistry(root, (state) => {
|
|
3011
3184
|
const migration = inspectLegacyMigration(root, state, now);
|
|
3012
3185
|
if (!migration.ok) return migration;
|
|
3013
|
-
const transport = state.hosts[host] ??=
|
|
3014
|
-
|
|
3015
|
-
if (transport.
|
|
3186
|
+
const transport = state.hosts[host] ??= emptyIdentityTransport();
|
|
3187
|
+
transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, now, kill));
|
|
3188
|
+
if (transport.throttle.paused) return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
|
|
3189
|
+
if (transport.permits.some((permit) => permit.kind !== "legacy") || transport.waiters.length) return { ok: false, reason: "identity-busy", retryAt: now + 1000 };
|
|
3016
3190
|
const attempts = Object.values(state.attempts);
|
|
3017
3191
|
const recent = attempts.filter((attempt) => attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
|
|
3018
3192
|
const own = recent.filter((attempt) => attempt.credentialKey === credentialKey && attempt.resource === resource && attempt.exceptional);
|
|
@@ -3027,7 +3201,7 @@ function claimIdentityBootstrap(root, { credentialKey, host, resource = "core",
|
|
|
3027
3201
|
const nonce = governorId();
|
|
3028
3202
|
state.attempts[id] = { credentialKey, host, resource, startedAt: now, retryAt: now + attemptRetryDelayMs(own.length),
|
|
3029
3203
|
ownerPid: process.pid, nonce, status: "started", quotaKey: null, resetMs: null, accounted: false, imported: false, exceptional: true };
|
|
3030
|
-
transport.
|
|
3204
|
+
transport.permits.push({ pid: process.pid, nonce, startedAt: now, kind: "bootstrap", bootId: null });
|
|
3031
3205
|
transport.lastStartedAt = now;
|
|
3032
3206
|
return { ok: true, value: { id, nonce } };
|
|
3033
3207
|
}, { now, kill });
|
|
@@ -3045,32 +3219,31 @@ function importIdentityDebts(root, state, credentialKey, identity, now) {
|
|
|
3045
3219
|
return withGovernorLock(scope, () => {
|
|
3046
3220
|
const loaded = readIdentityQuotaState(state, scope, now);
|
|
3047
3221
|
if (!loaded.ok) return loaded;
|
|
3222
|
+
for (const id of Object.keys(loaded.value.importMarkers)) {
|
|
3223
|
+
const attempt = state.attempts[id];
|
|
3224
|
+
if (!attempt || attempt.imported && attempt.quotaKey === identity.quotaKey) {
|
|
3225
|
+
delete loaded.value.importMarkers[id];
|
|
3226
|
+
}
|
|
3227
|
+
}
|
|
3048
3228
|
const transferred = [];
|
|
3049
3229
|
for (const [id, attempt] of Object.entries(state.attempts)) {
|
|
3050
3230
|
if (attempt.credentialKey !== credentialKey || attempt.imported) continue;
|
|
3051
|
-
const
|
|
3052
|
-
loaded.value.reservations[reservationId] ??= {
|
|
3231
|
+
const legacyControl = {
|
|
3053
3232
|
leaseId: id, intentId: id, costs: { core: 0, graphql: 0, [attempt.resource]: 1 }, actualCosts: null,
|
|
3054
3233
|
accountedCosts: { core: 0, graphql: 0, [attempt.resource]: attempt.accounted ? 1 : 0 },
|
|
3055
3234
|
notBefore: attempt.startedAt, status: "started", epochs: { core: null, graphql: null },
|
|
3056
3235
|
startedAt: attempt.startedAt, completedAt: null, outcome: null,
|
|
3057
3236
|
};
|
|
3237
|
+
if (!loaded.value.importMarkers[id] &&
|
|
3238
|
+
!transferReceiptToDebt(loaded.value, legacyControl, now, id)) {
|
|
3239
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
3240
|
+
}
|
|
3241
|
+
if (Object.keys(loaded.value.importMarkers).length >= IDENTITY_MAX_ATTEMPTS &&
|
|
3242
|
+
!loaded.value.importMarkers[id]) return { ok: false, reason: "identity-capacity" };
|
|
3243
|
+
loaded.value.importMarkers[id] = true;
|
|
3058
3244
|
transferred.push(id);
|
|
3059
3245
|
}
|
|
3060
|
-
|
|
3061
|
-
// that records the mapping commits. A crash in between leaves a charged
|
|
3062
|
-
// receipt here whose attempt is gone from the registry, and nothing else
|
|
3063
|
-
// ever looks at it again -- it presents only as a permanently smaller
|
|
3064
|
-
// budget. Re-bootstrapping the same credential is exactly when it can be
|
|
3065
|
-
// recognised, and a full window since it started proves a reset covered it.
|
|
3066
|
-
for (const key of Object.keys(loaded.value.reservations)) {
|
|
3067
|
-
const attemptId = /^reservation:(.+)$/.exec(key)?.[1];
|
|
3068
|
-
if (!attemptId || state.attempts[attemptId]) continue;
|
|
3069
|
-
const reservation = loaded.value.reservations[key];
|
|
3070
|
-
if (reservation.startedAt + IDENTITY_UNCERTAIN_MAX_MS > now) continue;
|
|
3071
|
-
delete loaded.value.reservations[key];
|
|
3072
|
-
}
|
|
3073
|
-
const written = writeGovernorState(scope.path, loaded.value);
|
|
3246
|
+
const written = writeGovernorQuotaState(scope.path, loaded.value, now);
|
|
3074
3247
|
if (!written.ok) return written;
|
|
3075
3248
|
// Marked transferred only once the receipts are durable. withIdentityRegistry
|
|
3076
3249
|
// persists the registry whatever this returns, so setting these before the
|
|
@@ -3090,7 +3263,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
|
|
|
3090
3263
|
const attempt = state.attempts[id];
|
|
3091
3264
|
if (!attempt || attempt.nonce !== nonce || attempt.credentialKey !== credentialKey) return { ok: false, reason: "stale" };
|
|
3092
3265
|
const transport = state.hosts[attempt.host];
|
|
3093
|
-
|
|
3266
|
+
transport.permits = transport.permits.filter((permit) => permit.nonce !== nonce);
|
|
3094
3267
|
attempt.status = response ? "finished" : "uncertain";
|
|
3095
3268
|
const parsed = response?.body;
|
|
3096
3269
|
const evidence = response?.rateLimit;
|
|
@@ -3128,7 +3301,7 @@ function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = nu
|
|
|
3128
3301
|
ledger.value.epochs.core = observation.budget.epoch;
|
|
3129
3302
|
ledger.value.observers.core = { etag: typeof response.etag === "string" ? response.etag : null, outcome: "healthy", at: now, nextAt: now + BUDGET_PROBE_MS };
|
|
3130
3303
|
}
|
|
3131
|
-
return
|
|
3304
|
+
return writeGovernorQuotaState(scope.path, ledger.value, now);
|
|
3132
3305
|
});
|
|
3133
3306
|
if (!seeded.ok) return seeded;
|
|
3134
3307
|
}
|
|
@@ -3267,35 +3440,53 @@ function startIdentityControl(coordinator, now = Date.now()) {
|
|
|
3267
3440
|
const exceptional = !budget || now >= budget.resetMs + BUDGET_RESET_GRACE_MS;
|
|
3268
3441
|
if (budget && now >= budget.resetMs && !exceptional) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
|
|
3269
3442
|
if (!exceptional) {
|
|
3270
|
-
const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now),
|
|
3443
|
+
const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now),
|
|
3444
|
+
state.debt.core.unresolvedUnits + state.debt.core.quiescentUnits +
|
|
3445
|
+
reservationCost(state.controlReceipts.core, "core", state.leases, now));
|
|
3271
3446
|
if (budget.remaining - resourceReserve(budget.limit) - reserved < 1) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
|
|
3272
3447
|
}
|
|
3273
3448
|
const recent = Object.values(registry.attempts).filter((attempt) => attempt.credentialKey === identity.credentialKey && attempt.resource === "core" && attempt.exceptional && attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
|
|
3274
3449
|
const retryAt = Math.max(0, ...recent.map((attempt) => attempt.retryAt), recent.length >= IDENTITY_ATTEMPT_LIMIT ? Math.min(...recent.map((attempt) => attempt.startedAt)) + IDENTITY_ATTEMPT_WINDOW_MS : 0);
|
|
3275
3450
|
if (exceptional && retryAt > now) return { ok: false, reason: "identity-backoff", retryAt };
|
|
3276
|
-
if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS
|
|
3451
|
+
if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS) return { ok: false, reason: "identity-capacity" };
|
|
3452
|
+
const priorControl = state.controlReceipts.core;
|
|
3453
|
+
if (priorControl) {
|
|
3454
|
+
if (!retireControlReceiptToDebt(state, "core", now)) return { ok: false, reason: "accounting-overflow" };
|
|
3455
|
+
}
|
|
3277
3456
|
const id = governorId();
|
|
3278
3457
|
registry.attempts[id] = { credentialKey: identity.credentialKey, host: identity.host, resource: "core", startedAt: now,
|
|
3279
3458
|
retryAt: exceptional ? now + attemptRetryDelayMs(recent.length) : now, ownerPid: process.pid, nonce: governorId(), status: "started",
|
|
3280
3459
|
quotaKey: identity.quotaKey, resetMs: budget?.resetMs ?? null, accounted: false, imported: true, exceptional };
|
|
3281
|
-
state.
|
|
3460
|
+
state.controlReceipts.core = { leaseId: id, intentId: id, costs: { core: 1, graphql: 0 }, actualCosts: null,
|
|
3282
3461
|
accountedCosts: { core: 0, graphql: 0 }, notBefore: now, status: "started", epochs: { core: budget?.epoch ?? null, graphql: null },
|
|
3283
|
-
startedAt: now, completedAt: null, outcome: null
|
|
3284
|
-
|
|
3285
|
-
|
|
3462
|
+
startedAt: now, completedAt: null, outcome: null,
|
|
3463
|
+
receipt: {
|
|
3464
|
+
scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
|
|
3465
|
+
ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
|
|
3466
|
+
deadline: now + ACQUISITION_STARTED_DEADLINE_MS,
|
|
3467
|
+
allowance: { core: 1, graphql: 0 }, dispatches: [],
|
|
3468
|
+
} };
|
|
3469
|
+
const written = writeGovernorQuotaState(scope.path, state, now);
|
|
3470
|
+
return written.ok ? { ok: true, value: { id, identity,
|
|
3471
|
+
receiptCapability: { reservationId: `control:core`, controlResource: "core",
|
|
3472
|
+
leaseId: id, scopeHash: scope.hash,
|
|
3473
|
+
ownerNonce: state.controlReceipts.core.receipt.ownerNonce,
|
|
3474
|
+
generation: state.controlReceipts.core.receipt.generation,
|
|
3475
|
+
deadline: state.controlReceipts.core.receipt.deadline } } } : written;
|
|
3286
3476
|
});
|
|
3287
3477
|
}, { now });
|
|
3288
3478
|
}
|
|
3289
3479
|
|
|
3290
|
-
function settleIdentityControl(coordinator, control, stdout, now = Date.now()
|
|
3480
|
+
function settleIdentityControl(coordinator, control, stdout, now = Date.now(),
|
|
3481
|
+
{ neverStarted = false } = {}) {
|
|
3291
3482
|
return withIdentityRegistry(coordinator.root, (registry) => {
|
|
3292
3483
|
const attempt = registry.attempts[control.id];
|
|
3293
3484
|
if (!attempt) return { ok: false, reason: "stale" };
|
|
3294
3485
|
const parsed = parseGhApiResponse(stdout ?? "");
|
|
3295
3486
|
const evidence = pickRateLimit(parsed.headers);
|
|
3296
3487
|
const proven = [200, 304].includes(parsed.status) && evidence?.resource === "core";
|
|
3297
|
-
attempt.status = proven ? "finished" : "uncertain";
|
|
3298
|
-
attempt.accounted = proven;
|
|
3488
|
+
attempt.status = proven || neverStarted ? "finished" : "uncertain";
|
|
3489
|
+
attempt.accounted = proven || neverStarted;
|
|
3299
3490
|
if (evidence?.resource === "core") {
|
|
3300
3491
|
attempt.resetMs = evidence.resetMs;
|
|
3301
3492
|
if (evidence.remaining === 0) attempt.retryAt = Math.max(attempt.retryAt, evidence.resetMs + BUDGET_RESET_GRACE_MS);
|
|
@@ -3304,64 +3495,93 @@ function settleIdentityControl(coordinator, control, stdout, now = Date.now()) {
|
|
|
3304
3495
|
return withGovernorLock(scope, () => {
|
|
3305
3496
|
const loaded = readIdentityQuotaState(registry, scope, now);
|
|
3306
3497
|
if (!loaded.ok) return loaded;
|
|
3307
|
-
const reservation = loaded.value.
|
|
3498
|
+
const reservation = loaded.value.controlReceipts.core?.intentId === control.id
|
|
3499
|
+
? loaded.value.controlReceipts.core : null;
|
|
3308
3500
|
if (!reservation) return { ok: false, reason: "stale" };
|
|
3309
|
-
if (proven) {
|
|
3310
|
-
const cost = parsed.status === 304 ? 0 : 1;
|
|
3501
|
+
if (proven || neverStarted) {
|
|
3502
|
+
const cost = neverStarted || parsed.status === 304 ? 0 : 1;
|
|
3311
3503
|
reservation.status = "completed";
|
|
3312
3504
|
reservation.completedAt = now;
|
|
3313
3505
|
reservation.outcome = "measured-success";
|
|
3314
3506
|
reservation.actualCosts = { core: cost, graphql: 0 };
|
|
3315
3507
|
reservation.accountedCosts = { core: cost, graphql: 0 };
|
|
3508
|
+
loaded.value.controlReceipts.core = null;
|
|
3509
|
+
} else if (!retireControlReceiptToDebt(loaded.value, "core", now)) {
|
|
3510
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
3316
3511
|
}
|
|
3317
|
-
return
|
|
3512
|
+
return writeGovernorQuotaState(scope.path, loaded.value, now);
|
|
3318
3513
|
});
|
|
3319
3514
|
}, { now });
|
|
3320
3515
|
}
|
|
3321
3516
|
|
|
3517
|
+
const HTTP_PERMIT_FAILURE_HOLDS = Object.freeze({
|
|
3518
|
+
"transport-busy": "coordination",
|
|
3519
|
+
"throttle-paused": "secondary",
|
|
3520
|
+
});
|
|
3521
|
+
|
|
3522
|
+
function httpPermitFailure(reason) {
|
|
3523
|
+
const error = new Error(identityCoordinationMessage(reason));
|
|
3524
|
+
if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, reason)) error.coordinationReason = reason;
|
|
3525
|
+
return error;
|
|
3526
|
+
}
|
|
3527
|
+
|
|
3322
3528
|
async function acquireIdentityHttpPermit(coordinator, {
|
|
3323
3529
|
signal,
|
|
3324
3530
|
now = Date.now,
|
|
3325
3531
|
wait = (delay) => new Promise((resolve) => setTimeout(resolve, delay)),
|
|
3532
|
+
operationDeadline,
|
|
3326
3533
|
} = {}) {
|
|
3327
3534
|
const flushed = await coordinator.flushCompletions?.();
|
|
3328
|
-
if (flushed && !flushed.ok) throw
|
|
3535
|
+
if (flushed && !flushed.ok) throw httpPermitFailure(flushed.reason);
|
|
3329
3536
|
const identity = coordinator.current();
|
|
3330
3537
|
if (!identity) throw new Error("Verified identity unavailable");
|
|
3331
3538
|
const queuedAt = now();
|
|
3332
|
-
const deadline =
|
|
3539
|
+
const deadline = Number.isFinite(operationDeadline)
|
|
3540
|
+
? Math.min(operationDeadline, queuedAt + ACQUISITION_STARTED_DEADLINE_MS)
|
|
3541
|
+
: queuedAt + GH_TIMEOUT_MS;
|
|
3333
3542
|
const nonce = governorId();
|
|
3334
3543
|
let acquired = false;
|
|
3335
3544
|
try {
|
|
3336
3545
|
for (;;) {
|
|
3337
3546
|
if (signal?.aborted) throw new Error("Request cancelled");
|
|
3338
3547
|
if (coordinator.current()?.accessKey !== identity.accessKey) throw new Error("Credential changed before request start");
|
|
3339
|
-
if (now() >= deadline) throw
|
|
3548
|
+
if (now() >= deadline) throw httpPermitFailure("transport-busy");
|
|
3340
3549
|
const result = withIdentityRegistry(coordinator.root, (state) => {
|
|
3341
3550
|
const migration = inspectLegacyMigration(coordinator.root, state, now());
|
|
3342
3551
|
if (!migration.ok) return migration;
|
|
3343
|
-
|
|
3344
|
-
|
|
3552
|
+
const permitAt = now();
|
|
3553
|
+
if (permitAt >= deadline) return { ok: false, reason: "transport-busy", retryAt: deadline };
|
|
3554
|
+
const transport = state.hosts[identity.host] ??= emptyIdentityTransport();
|
|
3345
3555
|
// Five consecutive throttles means the ladder has stopped being useful.
|
|
3346
3556
|
// Waiting out the deadline is what produced the last one, so the pause
|
|
3347
3557
|
// holds until a person retries or a primary reset opens a recovery.
|
|
3348
3558
|
if (transport.throttle?.paused) {
|
|
3349
3559
|
return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
|
|
3350
3560
|
}
|
|
3351
|
-
|
|
3561
|
+
transport.permits = transport.permits.filter((permit) => !staleHttpPermit(permit, permitAt));
|
|
3352
3562
|
// Admission can happen during the mandatory start gap. Persist call
|
|
3353
3563
|
// order once so later polling cannot overtake an earlier caller.
|
|
3354
|
-
|
|
3564
|
+
let waiter = transport.waiters.find((item) => item.nonce === nonce);
|
|
3565
|
+
if (!waiter) {
|
|
3355
3566
|
if (transport.waiters.length >= HTTP_MAX_WAITERS) return { ok: false, reason: "identity-capacity" };
|
|
3356
|
-
|
|
3567
|
+
waiter = { pid: process.pid, nonce, queuedAt: permitAt, deadline: Math.min(deadline, permitAt + GH_TIMEOUT_MS) };
|
|
3568
|
+
transport.waiters.push(waiter);
|
|
3569
|
+
} else if (waiter.deadline - permitAt <= GH_TIMEOUT_MS / 2 && waiter.deadline < deadline) {
|
|
3570
|
+
// Keep the existing registry's short waiter lease and FIFO position.
|
|
3571
|
+
// Renewing this lease cannot extend the admitted operation deadline
|
|
3572
|
+
// above, or the separate timeout of a child that actually starts.
|
|
3573
|
+
waiter.queuedAt = permitAt;
|
|
3574
|
+
waiter.deadline = Math.min(deadline, waiter.queuedAt + GH_TIMEOUT_MS);
|
|
3357
3575
|
}
|
|
3358
3576
|
const retryAt = Math.max(transport.cooldownUntil, transport.lastStartedAt + HTTP_START_GAP_MS);
|
|
3359
|
-
if (transport.
|
|
3360
|
-
|
|
3577
|
+
if (transport.permits.length >= HTTP_MAX_PERMITS ||
|
|
3578
|
+
transport.permits.some((permit) => permit.kind === "bootstrap") ||
|
|
3579
|
+
transport.waiters[0].nonce !== nonce || retryAt > permitAt) {
|
|
3580
|
+
return { ok: false, reason: "transport-busy", retryAt: Math.max(retryAt, permitAt + 50) };
|
|
3361
3581
|
}
|
|
3362
3582
|
transport.waiters.shift();
|
|
3363
|
-
transport.
|
|
3364
|
-
transport.lastStartedAt =
|
|
3583
|
+
transport.permits.push({ pid: process.pid, nonce, startedAt: permitAt, kind: "request", bootId: null });
|
|
3584
|
+
transport.lastStartedAt = permitAt;
|
|
3365
3585
|
return { ok: true, value: { nonce, host: identity.host } };
|
|
3366
3586
|
}, { now: now() });
|
|
3367
3587
|
if (result.ok) { acquired = true; return result.value; }
|
|
@@ -3370,7 +3590,7 @@ async function acquireIdentityHttpPermit(coordinator, {
|
|
|
3370
3590
|
// retryAt, so give the rest the poll interval the loop already uses.
|
|
3371
3591
|
const retryAt = result.retryAt ?? now() + 50;
|
|
3372
3592
|
if ((result.reason !== "transport-busy" && !retryableCoordination(result.reason)) ||
|
|
3373
|
-
now() >= deadline || retryAt >= deadline) throw
|
|
3593
|
+
now() >= deadline || retryAt >= deadline) throw httpPermitFailure(result.reason);
|
|
3374
3594
|
await wait(Math.max(1, Math.min(50, retryAt - now())));
|
|
3375
3595
|
}
|
|
3376
3596
|
} finally {
|
|
@@ -3454,6 +3674,10 @@ function transportCooldownDeadline({ retryAfter, status, secondary, at, attempts
|
|
|
3454
3674
|
return status === 429 || secondary === true ? at + throttleLadderMs(attempts) : null;
|
|
3455
3675
|
}
|
|
3456
3676
|
|
|
3677
|
+
function emptyIdentityTransport() {
|
|
3678
|
+
return { lastStartedAt: 0, cooldownUntil: 0, permits: [], waiters: [], throttle: emptyTransportThrottle() };
|
|
3679
|
+
}
|
|
3680
|
+
|
|
3457
3681
|
function emptyTransportThrottle() {
|
|
3458
3682
|
return { attempts: 0, lastAt: 0, paused: false };
|
|
3459
3683
|
}
|
|
@@ -3481,10 +3705,21 @@ function clearTransportThrottle(transport) {
|
|
|
3481
3705
|
}
|
|
3482
3706
|
}
|
|
3483
3707
|
|
|
3708
|
+
// Completion retries keep their original response time and become no-ops
|
|
3709
|
+
// after a successful durable write. Weak keys do not accumulate tombstones.
|
|
3710
|
+
const identityPermitCompletions = new WeakMap();
|
|
3484
3711
|
function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date.now()) {
|
|
3485
|
-
|
|
3712
|
+
let completion = identityPermitCompletions.get(permit);
|
|
3713
|
+
if (!completion) {
|
|
3714
|
+
completion = { at: now, done: false };
|
|
3715
|
+
identityPermitCompletions.set(permit, completion);
|
|
3716
|
+
}
|
|
3717
|
+
if (completion.done) return { ok: true };
|
|
3718
|
+
now = completion.at;
|
|
3719
|
+
const result = withIdentityRegistry(coordinator.root, (state) => {
|
|
3486
3720
|
const transport = state.hosts[permit.host];
|
|
3487
|
-
|
|
3721
|
+
const held = transport?.permits.find((item) => item.nonce === permit.nonce);
|
|
3722
|
+
if (transport) transport.permits = transport.permits.filter((item) => item.nonce !== permit.nonce);
|
|
3488
3723
|
if (transport && error) {
|
|
3489
3724
|
const response = parseGhApiResponse(error.stdout ?? "");
|
|
3490
3725
|
const verdict = classifyThrottle({
|
|
@@ -3503,11 +3738,14 @@ function releaseIdentityHttpPermit(coordinator, permit, error = null, now = Date
|
|
|
3503
3738
|
at: now,
|
|
3504
3739
|
});
|
|
3505
3740
|
}
|
|
3506
|
-
} else if (transport && !error
|
|
3741
|
+
} else if (transport && !error && held && held.startedAt > transport.throttle.lastAt &&
|
|
3742
|
+
held.startedAt >= transport.cooldownUntil) {
|
|
3507
3743
|
clearTransportThrottle(transport);
|
|
3508
3744
|
}
|
|
3509
3745
|
return { ok: true };
|
|
3510
3746
|
});
|
|
3747
|
+
if (result.ok) completion.done = true;
|
|
3748
|
+
return result;
|
|
3511
3749
|
}
|
|
3512
3750
|
|
|
3513
3751
|
// The explicit retry a paused transport waits for. Clears the ladder and the
|
|
@@ -3558,11 +3796,11 @@ const GOVERNOR_SCOPE_VERSION = 1;
|
|
|
3558
3796
|
// 5: scheduling fairness is persisted, because the starvation it prevents
|
|
3559
3797
|
// happens *between* planning passes -- each manual refresh is its own pass, so
|
|
3560
3798
|
// a counter that lives only inside one cannot see a run of them.
|
|
3561
|
-
const GOVERNOR_STATE_VERSION =
|
|
3799
|
+
const GOVERNOR_STATE_VERSION = 7;
|
|
3562
3800
|
// Readable-as-evidence, never written: the shapes a still-running older pane
|
|
3563
3801
|
// holds. Recognising that such a pane owns a live lease is what the restart
|
|
3564
3802
|
// boundary depends on.
|
|
3565
|
-
const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 5, 4, 3, 2];
|
|
3803
|
+
const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 6, 5, 4, 3, 2];
|
|
3566
3804
|
// The first version whose file is already in the per-resource observer shape.
|
|
3567
3805
|
// Anything below it is adapted before it can be read; see readLegacyGovernorState.
|
|
3568
3806
|
const GOVERNOR_OBSERVER_SPLIT_VERSION = 4;
|
|
@@ -3582,6 +3820,8 @@ const MANUAL_GRANT_STREAK_LIMIT = 3;
|
|
|
3582
3820
|
const GOVERNOR_MAX_LEASES = 128;
|
|
3583
3821
|
const GOVERNOR_MAX_INTENTS = 512;
|
|
3584
3822
|
const GOVERNOR_MAX_RESERVATIONS = 512;
|
|
3823
|
+
const GOVERNOR_MAX_DEBT_GROUPS = 128;
|
|
3824
|
+
const GOVERNOR_MAX_LEDGER_BYTES = 2 * 1024 * 1024;
|
|
3585
3825
|
const GOVERNOR_LOCK_WAIT_MS = 250;
|
|
3586
3826
|
// How old an unreadable lock record may be before it counts as abandoned. A
|
|
3587
3827
|
// creator owns the path from open("wx") until its owner record is written, a
|
|
@@ -3697,6 +3937,7 @@ function currentGovernorScope(scope) {
|
|
|
3697
3937
|
function emptyGovernorState() {
|
|
3698
3938
|
return {
|
|
3699
3939
|
version: GOVERNOR_STATE_VERSION,
|
|
3940
|
+
revision: 0,
|
|
3700
3941
|
epochs: { core: null, graphql: null },
|
|
3701
3942
|
budgets: {},
|
|
3702
3943
|
fairness: { manualStreak: 0 },
|
|
@@ -3712,6 +3953,14 @@ function emptyGovernorState() {
|
|
|
3712
3953
|
leases: {},
|
|
3713
3954
|
intents: {},
|
|
3714
3955
|
reservations: {},
|
|
3956
|
+
controlReceipts: { core: null, graphql: null },
|
|
3957
|
+
debt: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, {
|
|
3958
|
+
unresolvedUnits: 0, unresolvedCount: 0, quiescentUnits: 0, quiescentCount: 0,
|
|
3959
|
+
revision: 0,
|
|
3960
|
+
}])),
|
|
3961
|
+
debtGroups: {},
|
|
3962
|
+
ownerGenerations: {},
|
|
3963
|
+
importMarkers: {},
|
|
3715
3964
|
manualProbe: null,
|
|
3716
3965
|
};
|
|
3717
3966
|
}
|
|
@@ -3804,10 +4053,11 @@ function normalizeGovernorIntent(raw, nowMs) {
|
|
|
3804
4053
|
return { ...raw, costs };
|
|
3805
4054
|
}
|
|
3806
4055
|
|
|
3807
|
-
function normalizeGovernorReservation(raw, nowMs) {
|
|
4056
|
+
function normalizeGovernorReservation(raw, nowMs, { legacy = false } = {}) {
|
|
3808
4057
|
if (!exactKeys(raw, [
|
|
3809
4058
|
"leaseId", "intentId", "costs", "actualCosts", "accountedCosts", "notBefore", "status", "epochs",
|
|
3810
4059
|
"startedAt", "completedAt", "outcome",
|
|
4060
|
+
...(!legacy ? ["receipt"] : []),
|
|
3811
4061
|
])) return null;
|
|
3812
4062
|
const costs = exactResourceCosts(raw.costs);
|
|
3813
4063
|
const actualCosts = raw.actualCosts === null ? null : exactResourceCosts(raw.actualCosts);
|
|
@@ -3829,13 +4079,46 @@ function normalizeGovernorReservation(raw, nowMs) {
|
|
|
3829
4079
|
) return null;
|
|
3830
4080
|
if (actualCosts && RATE_RESOURCES.some((resource) => actualCosts[resource] > costs[resource])) return null;
|
|
3831
4081
|
if (RATE_RESOURCES.some((resource) => accountedCosts[resource] > (actualCosts?.[resource] ?? costs[resource]))) return null;
|
|
4082
|
+
if (!legacy && raw.receipt !== null) {
|
|
4083
|
+
const receipt = raw.receipt;
|
|
4084
|
+
if (!exactKeys(receipt, [
|
|
4085
|
+
"scopeHash", "ownerNonce", "generation", "ownerPid", "ownerBirth", "bootId",
|
|
4086
|
+
"deadline", "allowance", "dispatches",
|
|
4087
|
+
]) || typeof receipt.scopeHash !== "string" || !/^[0-9a-f]{64}$/.test(receipt.scopeHash) ||
|
|
4088
|
+
!validGovernorId(receipt.ownerNonce) || !validGovernorId(receipt.generation) ||
|
|
4089
|
+
!Number.isSafeInteger(receipt.ownerPid) || receipt.ownerPid < 1 ||
|
|
4090
|
+
receipt.ownerBirth !== null && (typeof receipt.ownerBirth !== "string" || receipt.ownerBirth.length > 128) ||
|
|
4091
|
+
receipt.bootId !== null && (typeof receipt.bootId !== "string" || receipt.bootId.length > 128) ||
|
|
4092
|
+
finiteTimestamp(receipt.deadline, nowMs) === undefined ||
|
|
4093
|
+
!exactKeys(receipt.allowance, RATE_RESOURCES) ||
|
|
4094
|
+
RATE_RESOURCES.some((resource) => receipt.allowance[resource] !== costs[resource]) ||
|
|
4095
|
+
!Array.isArray(receipt.dispatches) || receipt.dispatches.length > 32) return null;
|
|
4096
|
+
const issued = { core: 0, graphql: 0 };
|
|
4097
|
+
for (const [index, dispatch] of receipt.dispatches.entries()) {
|
|
4098
|
+
if (!exactKeys(dispatch, ["sequence", "operation", "costs", "issuedAt", "terminalAt", "childPid", "childBirth", "neverStarted"]) ||
|
|
4099
|
+
dispatch.sequence !== index + 1 || !operationCost(dispatch.operation) ||
|
|
4100
|
+
!exactKeys(dispatch.costs, RATE_RESOURCES) ||
|
|
4101
|
+
RATE_RESOURCES.some((resource) => dispatch.costs[resource] !== operationCost(dispatch.operation)[resource]) ||
|
|
4102
|
+
finiteTimestamp(dispatch.issuedAt, nowMs) === undefined ||
|
|
4103
|
+
finiteTimestamp(dispatch.terminalAt, nowMs, { nullable: true }) === undefined ||
|
|
4104
|
+
dispatch.childPid !== null && (!Number.isSafeInteger(dispatch.childPid) || dispatch.childPid < 1) ||
|
|
4105
|
+
dispatch.childBirth !== null && (typeof dispatch.childBirth !== "string" || dispatch.childBirth.length > 128) ||
|
|
4106
|
+
typeof dispatch.neverStarted !== "boolean" ||
|
|
4107
|
+
dispatch.neverStarted && (dispatch.childPid !== null || dispatch.terminalAt === null)) return null;
|
|
4108
|
+
for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
|
|
4109
|
+
}
|
|
4110
|
+
if (RATE_RESOURCES.some((resource) => !Number.isSafeInteger(issued[resource]) ||
|
|
4111
|
+
issued[resource] > receipt.allowance[resource])) return null;
|
|
4112
|
+
} else if (!legacy && raw.status === "started") {
|
|
4113
|
+
return null;
|
|
4114
|
+
}
|
|
3832
4115
|
return { ...raw, costs, actualCosts, accountedCosts, epochs };
|
|
3833
4116
|
}
|
|
3834
4117
|
|
|
3835
4118
|
function normalizeResourceObserver(raw, nowMs) {
|
|
3836
4119
|
if (!exactKeys(raw, ["etag", "outcome", "at", "nextAt"])) return null;
|
|
3837
4120
|
if (
|
|
3838
|
-
raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0) ||
|
|
4121
|
+
raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0 || raw.etag.length > 512) ||
|
|
3839
4122
|
!GOVERNOR_PROBE_STATUSES.has(raw.outcome) ||
|
|
3840
4123
|
finiteTimestamp(raw.at, nowMs) === undefined || raw.at > nowMs ||
|
|
3841
4124
|
finiteTimestamp(raw.nextAt, nowMs) === undefined
|
|
@@ -3843,10 +4126,11 @@ function normalizeResourceObserver(raw, nowMs) {
|
|
|
3843
4126
|
return { ...raw };
|
|
3844
4127
|
}
|
|
3845
4128
|
|
|
3846
|
-
function normalizeProbeClaim(raw, nowMs) {
|
|
4129
|
+
function normalizeProbeClaim(raw, nowMs, { legacy = false } = {}) {
|
|
3847
4130
|
if (raw === null) return null;
|
|
3848
4131
|
if (!exactKeys(raw, [
|
|
3849
4132
|
"ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds",
|
|
4133
|
+
...(!legacy ? ["debtSnapshot"] : []),
|
|
3850
4134
|
])) {
|
|
3851
4135
|
return undefined;
|
|
3852
4136
|
}
|
|
@@ -3858,6 +4142,12 @@ function normalizeProbeClaim(raw, nowMs) {
|
|
|
3858
4142
|
!Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
|
|
3859
4143
|
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
|
|
3860
4144
|
) return undefined;
|
|
4145
|
+
const snapshot = raw.debtSnapshot;
|
|
4146
|
+
if (!legacy && (!exactKeys(snapshot, ["units", "count", "barrierAt", "revision"]) ||
|
|
4147
|
+
!Number.isSafeInteger(snapshot.units) || snapshot.units < 0 ||
|
|
4148
|
+
!Number.isSafeInteger(snapshot.count) || snapshot.count < 0 ||
|
|
4149
|
+
finiteTimestamp(snapshot.barrierAt, nowMs) === undefined ||
|
|
4150
|
+
!Number.isSafeInteger(snapshot.revision) || snapshot.revision < 0)) return undefined;
|
|
3861
4151
|
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
|
|
3862
4152
|
}
|
|
3863
4153
|
|
|
@@ -3878,9 +4168,11 @@ function normalizeManualProbe(raw, nowMs) {
|
|
|
3878
4168
|
// never publishes what it reads. Every other caller takes the default, so a
|
|
3879
4169
|
// version this build does not write is unreadable and fails closed.
|
|
3880
4170
|
function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVERNOR_STATE_VERSION } = {}) {
|
|
4171
|
+
const legacyShape = acceptVersion < 7;
|
|
3881
4172
|
if (!exactKeys(raw, [
|
|
3882
4173
|
"version", "epochs", "budgets", "fairness", "observers", "probeClaims", "leases",
|
|
3883
4174
|
"intents", "reservations", "manualProbe",
|
|
4175
|
+
...(!legacyShape ? ["revision", "controlReceipts", "debt", "debtGroups", "ownerGenerations", "importMarkers"] : []),
|
|
3884
4176
|
]) || raw.version !== acceptVersion) return null;
|
|
3885
4177
|
if (!exactKeys(raw.fairness, ["manualStreak"]) ||
|
|
3886
4178
|
!Number.isSafeInteger(raw.fairness.manualStreak) || raw.fairness.manualStreak < 0) return null;
|
|
@@ -3894,6 +4186,75 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3894
4186
|
) return null;
|
|
3895
4187
|
|
|
3896
4188
|
const state = emptyGovernorState();
|
|
4189
|
+
state.version = acceptVersion;
|
|
4190
|
+
if (legacyShape) {
|
|
4191
|
+
delete state.revision;
|
|
4192
|
+
delete state.controlReceipts;
|
|
4193
|
+
delete state.debt;
|
|
4194
|
+
delete state.debtGroups;
|
|
4195
|
+
delete state.ownerGenerations;
|
|
4196
|
+
delete state.importMarkers;
|
|
4197
|
+
} else {
|
|
4198
|
+
if (!Number.isSafeInteger(raw.revision) || raw.revision < 0) return null;
|
|
4199
|
+
state.revision = raw.revision;
|
|
4200
|
+
if (!exactKeys(raw.controlReceipts, RATE_RESOURCES) || !exactKeys(raw.debt, RATE_RESOURCES) ||
|
|
4201
|
+
!isRecord(raw.debtGroups) || Object.keys(raw.debtGroups).length > GOVERNOR_MAX_DEBT_GROUPS ||
|
|
4202
|
+
!isRecord(raw.ownerGenerations) ||
|
|
4203
|
+
Object.keys(raw.ownerGenerations).length > GOVERNOR_MAX_LEASES + GOVERNOR_MAX_RESERVATIONS ||
|
|
4204
|
+
!isRecord(raw.importMarkers) || Object.keys(raw.importMarkers).length > IDENTITY_MAX_ATTEMPTS) return null;
|
|
4205
|
+
for (const [id, marker] of Object.entries(raw.importMarkers)) {
|
|
4206
|
+
if (!validGovernorId(id) || marker !== true) return null;
|
|
4207
|
+
state.importMarkers[id] = true;
|
|
4208
|
+
}
|
|
4209
|
+
for (const [leaseId, generation] of Object.entries(raw.ownerGenerations)) {
|
|
4210
|
+
if (!validGovernorId(leaseId) || !validGovernorId(generation)) return null;
|
|
4211
|
+
state.ownerGenerations[leaseId] = generation;
|
|
4212
|
+
}
|
|
4213
|
+
for (const resource of RATE_RESOURCES) {
|
|
4214
|
+
const debt = raw.debt[resource];
|
|
4215
|
+
if (!exactKeys(debt, ["unresolvedUnits", "unresolvedCount", "quiescentUnits", "quiescentCount", "revision"]) ||
|
|
4216
|
+
Object.values(debt).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
|
|
4217
|
+
state.debt[resource] = { ...debt };
|
|
4218
|
+
// Control receipts are normalized independently from the 512 data slots.
|
|
4219
|
+
if (raw.controlReceipts[resource] !== null) {
|
|
4220
|
+
const control = normalizeGovernorReservation(raw.controlReceipts[resource], nowMs);
|
|
4221
|
+
if (!control) return null;
|
|
4222
|
+
state.controlReceipts[resource] = control;
|
|
4223
|
+
}
|
|
4224
|
+
}
|
|
4225
|
+
for (const [id, group] of Object.entries(raw.debtGroups)) {
|
|
4226
|
+
if (!validGovernorId(id) || !exactKeys(group, [
|
|
4227
|
+
"nonce", "ownerNonce", "ownerPid", "ownerBirth", "bootId", "unknownChild", "sealed",
|
|
4228
|
+
"quiescent", "units", "counts", "at", "barrierAt", "barrierRevisions",
|
|
4229
|
+
]) || !validGovernorId(group.nonce) ||
|
|
4230
|
+
group.ownerNonce !== null && !validGovernorId(group.ownerNonce) ||
|
|
4231
|
+
group.ownerPid !== null && (!Number.isSafeInteger(group.ownerPid) || group.ownerPid < 1) ||
|
|
4232
|
+
group.ownerBirth !== null && (typeof group.ownerBirth !== "string" || group.ownerBirth.length > 128) ||
|
|
4233
|
+
group.bootId !== null && (typeof group.bootId !== "string" || group.bootId.length > 128) ||
|
|
4234
|
+
typeof group.unknownChild !== "boolean" || typeof group.sealed !== "boolean" ||
|
|
4235
|
+
typeof group.quiescent !== "boolean" || !exactKeys(group.units, RATE_RESOURCES) ||
|
|
4236
|
+
!exactKeys(group.counts, RATE_RESOURCES) ||
|
|
4237
|
+
[...Object.values(group.units), ...Object.values(group.counts)].some((value) =>
|
|
4238
|
+
!Number.isSafeInteger(value) || value < 0) ||
|
|
4239
|
+
finiteTimestamp(group.at, nowMs) === undefined ||
|
|
4240
|
+
finiteTimestamp(group.barrierAt, nowMs, { nullable: true }) === undefined ||
|
|
4241
|
+
!exactKeys(group.barrierRevisions, RATE_RESOURCES) ||
|
|
4242
|
+
Object.values(group.barrierRevisions).some((value) =>
|
|
4243
|
+
!Number.isSafeInteger(value) || value < 0)) return null;
|
|
4244
|
+
state.debtGroups[id] = structuredClone(group);
|
|
4245
|
+
}
|
|
4246
|
+
for (const resource of RATE_RESOURCES) {
|
|
4247
|
+
const totals = { unresolvedUnits: 0, unresolvedCount: 0,
|
|
4248
|
+
quiescentUnits: 0, quiescentCount: 0 };
|
|
4249
|
+
for (const group of Object.values(state.debtGroups)) {
|
|
4250
|
+
const prefix = group.quiescent ? "quiescent" : "unresolved";
|
|
4251
|
+
totals[`${prefix}Units`] += group.units[resource];
|
|
4252
|
+
totals[`${prefix}Count`] += group.counts[resource];
|
|
4253
|
+
}
|
|
4254
|
+
if (Object.entries(totals).some(([field, amount]) =>
|
|
4255
|
+
!Number.isSafeInteger(amount) || amount !== state.debt[resource][field])) return null;
|
|
4256
|
+
}
|
|
4257
|
+
}
|
|
3897
4258
|
state.epochs = { ...raw.epochs };
|
|
3898
4259
|
state.fairness = { ...raw.fairness };
|
|
3899
4260
|
for (const resource of RATE_RESOURCES) {
|
|
@@ -3922,7 +4283,7 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3922
4283
|
if (Object.keys(state.intents).length > GOVERNOR_MAX_INTENTS) return null;
|
|
3923
4284
|
for (const [id, rawReservation] of Object.entries(raw.reservations)) {
|
|
3924
4285
|
if (!id.startsWith("reservation:") || !validGovernorId(id.slice(12))) return null;
|
|
3925
|
-
const reservation = normalizeGovernorReservation(rawReservation, nowMs);
|
|
4286
|
+
const reservation = normalizeGovernorReservation(rawReservation, nowMs, { legacy: legacyShape });
|
|
3926
4287
|
if (!reservation || id !== `reservation:${reservation.intentId}`) return null;
|
|
3927
4288
|
if (
|
|
3928
4289
|
!prune || reservation.status === "started" || reservation.status === "completed" ||
|
|
@@ -3930,8 +4291,9 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3930
4291
|
) state.reservations[id] = reservation;
|
|
3931
4292
|
}
|
|
3932
4293
|
if (Object.keys(state.reservations).length > GOVERNOR_MAX_RESERVATIONS) return null;
|
|
4294
|
+
if (prune && !legacyShape) retireOwnerGenerations(state);
|
|
3933
4295
|
for (const resource of RATE_RESOURCES) {
|
|
3934
|
-
const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs);
|
|
4296
|
+
const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs, { legacy: legacyShape });
|
|
3935
4297
|
if (claim === undefined) return null;
|
|
3936
4298
|
state.probeClaims[resource] = claim?.leaseUntil > nowMs ? claim : null;
|
|
3937
4299
|
}
|
|
@@ -3940,6 +4302,17 @@ function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVE
|
|
|
3940
4302
|
return state;
|
|
3941
4303
|
}
|
|
3942
4304
|
|
|
4305
|
+
function retireOwnerGenerations(state) {
|
|
4306
|
+
const detailedOwners = new Set(Object.values(state.reservations)
|
|
4307
|
+
.filter((reservation) => reservation.receipt !== null)
|
|
4308
|
+
.map((reservation) => reservation.leaseId));
|
|
4309
|
+
for (const leaseId of Object.keys(state.ownerGenerations)) {
|
|
4310
|
+
if (!state.leases[leaseId] && !detailedOwners.has(leaseId)) {
|
|
4311
|
+
delete state.ownerGenerations[leaseId];
|
|
4312
|
+
}
|
|
4313
|
+
}
|
|
4314
|
+
}
|
|
4315
|
+
|
|
3943
4316
|
// Reads a still-running older pane's file as evidence. A version number alone
|
|
3944
4317
|
// is not enough to identify a document: versions 2 and 3 share a *shape* that
|
|
3945
4318
|
// version 4 replaced, so accepting their version against the current key set
|
|
@@ -4048,8 +4421,296 @@ function migrateGovernorState(raw, nowMs) {
|
|
|
4048
4421
|
: adaptPreSplitGovernorShape(raw);
|
|
4049
4422
|
if (!shaped) return null;
|
|
4050
4423
|
if (!isRecord(shaped.fairness)) shaped.fairness = { manualStreak: 0 };
|
|
4051
|
-
shaped.version =
|
|
4052
|
-
|
|
4424
|
+
shaped.version = 6;
|
|
4425
|
+
const legacy = normalizeGovernorState(dropSupersededTabCosts(shaped), nowMs, {
|
|
4426
|
+
prune: false,
|
|
4427
|
+
acceptVersion: 6,
|
|
4428
|
+
});
|
|
4429
|
+
return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
|
|
4430
|
+
}
|
|
4431
|
+
|
|
4432
|
+
function migrateLegacyGovernorReceipts(legacy, nowMs) {
|
|
4433
|
+
const state = { ...legacy, version: GOVERNOR_STATE_VERSION,
|
|
4434
|
+
revision: 0,
|
|
4435
|
+
controlReceipts: { core: null, graphql: null },
|
|
4436
|
+
debt: emptyGovernorState().debt,
|
|
4437
|
+
debtGroups: {},
|
|
4438
|
+
ownerGenerations: {},
|
|
4439
|
+
importMarkers: {},
|
|
4440
|
+
reservations: {},
|
|
4441
|
+
};
|
|
4442
|
+
state.probeClaims = { core: null, graphql: null };
|
|
4443
|
+
for (const resource of RATE_RESOURCES) {
|
|
4444
|
+
if (legacy.probeClaims[resource]) {
|
|
4445
|
+
state.observers[resource] = { ...state.observers[resource], outcome: "idle", nextAt: nowMs };
|
|
4446
|
+
}
|
|
4447
|
+
}
|
|
4448
|
+
const units = { core: 0, graphql: 0 };
|
|
4449
|
+
const counts = { core: 0, graphql: 0 };
|
|
4450
|
+
for (const reservation of Object.values(legacy.reservations)) {
|
|
4451
|
+
for (const resource of RATE_RESOURCES) {
|
|
4452
|
+
const charge = reservationCost(reservation, resource, legacy.leases, nowMs);
|
|
4453
|
+
if (!Number.isSafeInteger(charge) || !Number.isSafeInteger(units[resource] + charge)) return null;
|
|
4454
|
+
units[resource] += charge;
|
|
4455
|
+
if (charge > 0) counts[resource] += 1;
|
|
4456
|
+
}
|
|
4457
|
+
}
|
|
4458
|
+
if (RATE_RESOURCES.some((resource) => units[resource] > 0)) {
|
|
4459
|
+
const nonce = governorId();
|
|
4460
|
+
state.debtGroups[nonce] = {
|
|
4461
|
+
nonce, ownerNonce: null, ownerPid: null, ownerBirth: null, bootId: coordinationBootId(),
|
|
4462
|
+
unknownChild: true, sealed: true, quiescent: false,
|
|
4463
|
+
units, counts, at: nowMs, barrierAt: null,
|
|
4464
|
+
barrierRevisions: { core: 0, graphql: 0 },
|
|
4465
|
+
};
|
|
4466
|
+
for (const resource of RATE_RESOURCES) {
|
|
4467
|
+
state.debt[resource].unresolvedUnits = units[resource];
|
|
4468
|
+
state.debt[resource].unresolvedCount = counts[resource];
|
|
4469
|
+
state.debt[resource].revision = 1;
|
|
4470
|
+
}
|
|
4471
|
+
}
|
|
4472
|
+
return normalizeGovernorState(state, nowMs);
|
|
4473
|
+
}
|
|
4474
|
+
|
|
4475
|
+
const UNKNOWN_DEBT_GROUP_ID = "ffffffff-ffff-4fff-8fff-ffffffffffff";
|
|
4476
|
+
|
|
4477
|
+
let cachedCoordinationBootId;
|
|
4478
|
+
let cachedCoordinationProcessBirth;
|
|
4479
|
+
function coordinationBootId() {
|
|
4480
|
+
if (cachedCoordinationBootId !== undefined) return cachedCoordinationBootId;
|
|
4481
|
+
try {
|
|
4482
|
+
const value = process.platform === "linux"
|
|
4483
|
+
? readFileSync("/proc/sys/kernel/random/boot_id", "utf8").trim()
|
|
4484
|
+
: process.platform === "darwin"
|
|
4485
|
+
? execFileSync("/usr/sbin/sysctl", ["-n", "kern.boottime"], { encoding: "utf8", timeout: 1000 }).trim()
|
|
4486
|
+
: "";
|
|
4487
|
+
cachedCoordinationBootId = value.length > 0 && value.length <= 128 ? value : null;
|
|
4488
|
+
} catch {
|
|
4489
|
+
cachedCoordinationBootId = null;
|
|
4490
|
+
}
|
|
4491
|
+
return cachedCoordinationBootId;
|
|
4492
|
+
}
|
|
4493
|
+
|
|
4494
|
+
function coordinationProcessBirth(pid = process.pid) {
|
|
4495
|
+
if (pid === process.pid && cachedCoordinationProcessBirth !== undefined) {
|
|
4496
|
+
return cachedCoordinationProcessBirth;
|
|
4497
|
+
}
|
|
4498
|
+
let value = null;
|
|
4499
|
+
try {
|
|
4500
|
+
if (process.platform === "linux") {
|
|
4501
|
+
const stat = readFileSync(`/proc/${pid}/stat`, "utf8");
|
|
4502
|
+
const fields = stat.slice(stat.lastIndexOf(")") + 2).trim().split(/\s+/);
|
|
4503
|
+
value = fields[19] ?? null; // starttime is field 22, after pid and comm.
|
|
4504
|
+
} else if (process.platform === "darwin") {
|
|
4505
|
+
value = execFileSync("/bin/ps", ["-o", "lstart=", "-p", String(pid)], {
|
|
4506
|
+
encoding: "utf8", timeout: 1000,
|
|
4507
|
+
}).trim() || null;
|
|
4508
|
+
}
|
|
4509
|
+
} catch { /* an unavailable birth identity is explicitly unknown */ }
|
|
4510
|
+
if (value && value.length > 128) value = null;
|
|
4511
|
+
if (pid === process.pid) cachedCoordinationProcessBirth = value;
|
|
4512
|
+
return value;
|
|
4513
|
+
}
|
|
4514
|
+
|
|
4515
|
+
function debtGroupTarget(state, receipt, groupId = null) {
|
|
4516
|
+
const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
|
|
4517
|
+
const count = Object.keys(state.debtGroups).length;
|
|
4518
|
+
const owned = groupId ? state.debtGroups[groupId] : null;
|
|
4519
|
+
if (owned && !owned.quiescent && owned.bootId === incomingBootId &&
|
|
4520
|
+
(!receipt.receipt || owned.ownerNonce === null ||
|
|
4521
|
+
owned.ownerNonce === receipt.receipt.ownerNonce)) return groupId;
|
|
4522
|
+
if (groupId && !owned && count < GOVERNOR_MAX_DEBT_GROUPS - 1) return groupId;
|
|
4523
|
+
const shared = Object.entries(state.debtGroups).find(([, group]) =>
|
|
4524
|
+
!group.quiescent && group.ownerNonce === null && group.unknownChild &&
|
|
4525
|
+
group.bootId === incomingBootId);
|
|
4526
|
+
if (shared) return shared[0];
|
|
4527
|
+
if (count >= GOVERNOR_MAX_DEBT_GROUPS) return null;
|
|
4528
|
+
return state.debtGroups[UNKNOWN_DEBT_GROUP_ID] ? governorId() : UNKNOWN_DEBT_GROUP_ID;
|
|
4529
|
+
}
|
|
4530
|
+
|
|
4531
|
+
function transferReceiptToDebt(state, receipt, nowMs, groupId = null) {
|
|
4532
|
+
const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
4533
|
+
reservationCost(receipt, resource, state.leases, nowMs)]));
|
|
4534
|
+
if (RATE_RESOURCES.every((resource) => units[resource] === 0)) return true;
|
|
4535
|
+
const id = debtGroupTarget(state, receipt, groupId);
|
|
4536
|
+
if (id === null) return false;
|
|
4537
|
+
const incomingBootId = receipt.receipt?.bootId ?? coordinationBootId();
|
|
4538
|
+
const dedicated = id === groupId;
|
|
4539
|
+
const group = state.debtGroups[id] ?? {
|
|
4540
|
+
nonce: governorId(), ownerNonce: null, ownerPid: null, ownerBirth: null,
|
|
4541
|
+
bootId: incomingBootId,
|
|
4542
|
+
unknownChild: !dedicated || !receipt.receipt, sealed: true, quiescent: false,
|
|
4543
|
+
units: { core: 0, graphql: 0 }, counts: { core: 0, graphql: 0 },
|
|
4544
|
+
at: nowMs, barrierAt: null,
|
|
4545
|
+
barrierRevisions: { core: 0, graphql: 0 },
|
|
4546
|
+
};
|
|
4547
|
+
if (group.quiescent) return false;
|
|
4548
|
+
if (receipt.receipt && dedicated) {
|
|
4549
|
+
if (group.ownerNonce !== null && group.ownerNonce !== receipt.receipt.ownerNonce) return false;
|
|
4550
|
+
group.ownerNonce = receipt.receipt.ownerNonce;
|
|
4551
|
+
group.ownerPid = receipt.receipt.ownerPid;
|
|
4552
|
+
group.ownerBirth = receipt.receipt.ownerBirth;
|
|
4553
|
+
group.bootId = receipt.receipt.bootId;
|
|
4554
|
+
group.unknownChild ||= receipt.receipt.dispatches.some((dispatch) =>
|
|
4555
|
+
dispatch.terminalAt === null);
|
|
4556
|
+
}
|
|
4557
|
+
for (const resource of RATE_RESOURCES) {
|
|
4558
|
+
const amount = units[resource];
|
|
4559
|
+
if (!Number.isSafeInteger(amount) || amount < 0 ||
|
|
4560
|
+
!Number.isSafeInteger(state.debt[resource].unresolvedUnits + amount) ||
|
|
4561
|
+
!Number.isSafeInteger(group.units[resource] + amount) ||
|
|
4562
|
+
!Number.isSafeInteger(state.debt[resource].unresolvedCount + (amount > 0 ? 1 : 0)) ||
|
|
4563
|
+
!Number.isSafeInteger(state.debt[resource].revision + 1) ||
|
|
4564
|
+
!Number.isSafeInteger(group.counts[resource] + (amount > 0 ? 1 : 0))) return false;
|
|
4565
|
+
}
|
|
4566
|
+
for (const resource of RATE_RESOURCES) {
|
|
4567
|
+
const amount = units[resource];
|
|
4568
|
+
state.debt[resource].unresolvedUnits += amount;
|
|
4569
|
+
state.debt[resource].unresolvedCount += amount > 0 ? 1 : 0;
|
|
4570
|
+
state.debt[resource].revision += 1;
|
|
4571
|
+
group.units[resource] += amount;
|
|
4572
|
+
group.counts[resource] += amount > 0 ? 1 : 0;
|
|
4573
|
+
}
|
|
4574
|
+
// The newest charge, not the group's creation, starts the window after
|
|
4575
|
+
// which reconcileBootDebt may treat the group as settled.
|
|
4576
|
+
group.at = Math.max(group.at, nowMs);
|
|
4577
|
+
state.debtGroups[id] = group;
|
|
4578
|
+
return true;
|
|
4579
|
+
}
|
|
4580
|
+
|
|
4581
|
+
function makeDebtGroupQuiescent(state, group, at) {
|
|
4582
|
+
for (const resource of RATE_RESOURCES) {
|
|
4583
|
+
const debt = state.debt[resource];
|
|
4584
|
+
if (debt.unresolvedUnits < group.units[resource] || debt.unresolvedCount < group.counts[resource] ||
|
|
4585
|
+
!Number.isSafeInteger(debt.quiescentUnits + group.units[resource]) ||
|
|
4586
|
+
!Number.isSafeInteger(debt.quiescentCount + group.counts[resource]) ||
|
|
4587
|
+
!Number.isSafeInteger(debt.revision + 1)) return false;
|
|
4588
|
+
}
|
|
4589
|
+
for (const resource of RATE_RESOURCES) {
|
|
4590
|
+
const debt = state.debt[resource];
|
|
4591
|
+
debt.unresolvedUnits -= group.units[resource];
|
|
4592
|
+
debt.unresolvedCount -= group.counts[resource];
|
|
4593
|
+
debt.quiescentUnits += group.units[resource];
|
|
4594
|
+
debt.quiescentCount += group.counts[resource];
|
|
4595
|
+
debt.revision += 1;
|
|
4596
|
+
}
|
|
4597
|
+
group.quiescent = true;
|
|
4598
|
+
group.barrierAt = at;
|
|
4599
|
+
group.barrierRevisions = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
4600
|
+
resource, state.debt[resource].revision,
|
|
4601
|
+
]));
|
|
4602
|
+
return true;
|
|
4603
|
+
}
|
|
4604
|
+
|
|
4605
|
+
function retireControlReceiptToDebt(state, resource, at) {
|
|
4606
|
+
const control = state.controlReceipts[resource];
|
|
4607
|
+
if (!control) return true;
|
|
4608
|
+
// A control receipt without an issued dispatch cannot have started HTTP.
|
|
4609
|
+
// Removing it also fences a late owner before it can issue a child.
|
|
4610
|
+
if (control.receipt.dispatches.length === 0) {
|
|
4611
|
+
state.controlReceipts[resource] = null;
|
|
4612
|
+
return true;
|
|
4613
|
+
}
|
|
4614
|
+
const generation = control.receipt.generation;
|
|
4615
|
+
if (!transferReceiptToDebt(state, control, at, generation)) return false;
|
|
4616
|
+
const group = state.debtGroups[generation];
|
|
4617
|
+
if (group && !group.unknownChild && group.ownerPid !== null &&
|
|
4618
|
+
group.ownerBirth !== null && group.bootId !== null &&
|
|
4619
|
+
!makeDebtGroupQuiescent(state, group, at)) return false;
|
|
4620
|
+
state.controlReceipts[resource] = null;
|
|
4621
|
+
return true;
|
|
4622
|
+
}
|
|
4623
|
+
|
|
4624
|
+
// A reboot proves an unknown owner's children are gone. So does age: a group's
|
|
4625
|
+
// `at` is its newest charge, and once a full window has elapsed since then
|
|
4626
|
+
// GitHub has charged every covered request to a window that is already over.
|
|
4627
|
+
function reconcileBootDebt(state, at) {
|
|
4628
|
+
const bootId = coordinationBootId();
|
|
4629
|
+
let changed = false;
|
|
4630
|
+
for (const group of Object.values(state.debtGroups)) {
|
|
4631
|
+
if (group.quiescent) continue;
|
|
4632
|
+
const priorBoot = bootId !== null && group.bootId !== null && group.bootId !== bootId;
|
|
4633
|
+
const outlivedWindow = group.sealed && at - group.at >= IDENTITY_UNCERTAIN_MAX_MS;
|
|
4634
|
+
if (!priorBoot && !outlivedWindow) continue;
|
|
4635
|
+
if (!makeDebtGroupQuiescent(state, group, at)) {
|
|
4636
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4637
|
+
}
|
|
4638
|
+
changed = true;
|
|
4639
|
+
}
|
|
4640
|
+
return { ok: true, changed };
|
|
4641
|
+
}
|
|
4642
|
+
|
|
4643
|
+
function compactGovernorReceipts(state, nowMs) {
|
|
4644
|
+
const sealed = new Set();
|
|
4645
|
+
const full = Object.keys(state.reservations).length >= GOVERNOR_MAX_RESERVATIONS;
|
|
4646
|
+
let changed = false;
|
|
4647
|
+
const dueGenerations = new Set(Object.values(state.reservations)
|
|
4648
|
+
.filter((reservation) => reservation.receipt && (
|
|
4649
|
+
reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
|
|
4650
|
+
full && reservation.status === "completed"))
|
|
4651
|
+
.map((reservation) => reservation.receipt.generation));
|
|
4652
|
+
// A full debt journal must not prevent a claimed observer from retiring
|
|
4653
|
+
// older quiescent groups. Keep the detailed receipts until space is proven.
|
|
4654
|
+
const candidates = Object.values(state.reservations).filter((reservation) => {
|
|
4655
|
+
const groupId = reservation.receipt?.generation ?? null;
|
|
4656
|
+
return (full && reservation.status === "completed" && !reservation.receipt ||
|
|
4657
|
+
groupId && dueGenerations.has(groupId) &&
|
|
4658
|
+
!(reservation.status === "started" && reservation.receipt.dispatches.length === 0)) &&
|
|
4659
|
+
RATE_RESOURCES.some((resource) =>
|
|
4660
|
+
reservationCost(reservation, resource, state.leases, nowMs) > 0);
|
|
4661
|
+
});
|
|
4662
|
+
if (candidates.length > 0) {
|
|
4663
|
+
const trial = structuredClone(state);
|
|
4664
|
+
for (const reservation of candidates) {
|
|
4665
|
+
const groupId = reservation.receipt?.generation ?? null;
|
|
4666
|
+
if (debtGroupTarget(trial, reservation, groupId) === null) return { ok: true, changed: false };
|
|
4667
|
+
if (!transferReceiptToDebt(trial, reservation, nowMs, groupId)) {
|
|
4668
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4669
|
+
}
|
|
4670
|
+
}
|
|
4671
|
+
}
|
|
4672
|
+
if (full) {
|
|
4673
|
+
for (const [id, reservation] of Object.entries(state.reservations)) {
|
|
4674
|
+
if (reservation.status !== "completed" || reservation.receipt) continue;
|
|
4675
|
+
if (!transferReceiptToDebt(state, reservation, nowMs)) {
|
|
4676
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4677
|
+
}
|
|
4678
|
+
delete state.reservations[id];
|
|
4679
|
+
changed = true;
|
|
4680
|
+
}
|
|
4681
|
+
}
|
|
4682
|
+
for (const reservation of Object.values(state.reservations)) {
|
|
4683
|
+
if (reservation.receipt && (
|
|
4684
|
+
reservation.status === "started" && reservation.receipt.deadline <= nowMs ||
|
|
4685
|
+
full && reservation.status === "completed")) {
|
|
4686
|
+
sealed.add(reservation.receipt.generation);
|
|
4687
|
+
}
|
|
4688
|
+
}
|
|
4689
|
+
if (sealed.size === 0) {
|
|
4690
|
+
if (changed) retireOwnerGenerations(state);
|
|
4691
|
+
return { ok: true, changed };
|
|
4692
|
+
}
|
|
4693
|
+
// Seal the complete owner generation. A late completion of one receipt cannot
|
|
4694
|
+
// make another already-authorized child in that generation disappear.
|
|
4695
|
+
for (const generation of sealed) {
|
|
4696
|
+
for (const [id, reservation] of Object.entries(state.reservations)) {
|
|
4697
|
+
if (reservation.receipt?.generation !== generation) continue;
|
|
4698
|
+
if (!(reservation.status === "started" && reservation.receipt.dispatches.length === 0) &&
|
|
4699
|
+
!transferReceiptToDebt(state, reservation, nowMs, generation)) {
|
|
4700
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4701
|
+
}
|
|
4702
|
+
delete state.reservations[id];
|
|
4703
|
+
state.ownerGenerations[reservation.leaseId] = governorId();
|
|
4704
|
+
}
|
|
4705
|
+
const group = state.debtGroups[generation];
|
|
4706
|
+
if (group && !group.unknownChild && group.ownerPid !== null &&
|
|
4707
|
+
group.ownerBirth !== null && group.bootId !== null &&
|
|
4708
|
+
!makeDebtGroupQuiescent(state, group, nowMs)) {
|
|
4709
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4710
|
+
}
|
|
4711
|
+
}
|
|
4712
|
+
retireOwnerGenerations(state);
|
|
4713
|
+
return { ok: true, changed: true };
|
|
4053
4714
|
}
|
|
4054
4715
|
|
|
4055
4716
|
function migrateV1GovernorState(raw, nowMs) {
|
|
@@ -4058,7 +4719,7 @@ function migrateV1GovernorState(raw, nowMs) {
|
|
|
4058
4719
|
"intents", "reservations", "manualProbe",
|
|
4059
4720
|
]) || raw.version !== 1 || !isRecord(raw.budgets) || !isRecord(raw.reservations)) return null;
|
|
4060
4721
|
const migrated = structuredClone(raw);
|
|
4061
|
-
migrated.version =
|
|
4722
|
+
migrated.version = 6;
|
|
4062
4723
|
migrated.observers = { core: { etag: null, outcome: "idle", at: 0, nextAt: 0 } };
|
|
4063
4724
|
// The v1 core value came from /rate_limit, which is not authoritative. Do not
|
|
4064
4725
|
// preserve it as bootstrap evidence. GraphQL remains valid.
|
|
@@ -4086,7 +4747,8 @@ function migrateV1GovernorState(raw, nowMs) {
|
|
|
4086
4747
|
]));
|
|
4087
4748
|
migrated.probeClaims = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, null]));
|
|
4088
4749
|
migrated.fairness = { manualStreak: 0 };
|
|
4089
|
-
|
|
4750
|
+
const legacy = normalizeGovernorState(migrated, nowMs, { prune: false, acceptVersion: 6 });
|
|
4751
|
+
return legacy ? migrateLegacyGovernorReceipts(legacy, nowMs) : null;
|
|
4090
4752
|
}
|
|
4091
4753
|
|
|
4092
4754
|
function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
|
|
@@ -4099,11 +4761,21 @@ function readGovernorState(path, nowMs, { persistMigration = true } = {}) {
|
|
|
4099
4761
|
: { ok: false, reason: "unwritable" };
|
|
4100
4762
|
}
|
|
4101
4763
|
try {
|
|
4764
|
+
if (Buffer.byteLength(raw) > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "capacity" };
|
|
4102
4765
|
const parsed = JSON.parse(raw);
|
|
4103
4766
|
const normalized = normalizeGovernorState(parsed, nowMs);
|
|
4104
4767
|
if (normalized) return { ok: true, value: normalized };
|
|
4105
4768
|
const migrated = migrateGovernorState(parsed, nowMs);
|
|
4106
4769
|
if (!migrated) return { ok: false, reason: "corrupt" };
|
|
4770
|
+
if (persistMigration && parsed.version === 6) {
|
|
4771
|
+
const backupPath = `${path}.pre-v7.backup`;
|
|
4772
|
+
try {
|
|
4773
|
+
writeFileSync(backupPath, raw, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
4774
|
+
chmodSync(backupPath, 0o600);
|
|
4775
|
+
} catch (error) {
|
|
4776
|
+
if (error?.code !== "EEXIST") return { ok: false, reason: "unwritable" };
|
|
4777
|
+
}
|
|
4778
|
+
}
|
|
4107
4779
|
if (!persistMigration) return { ok: true, value: migrated, migrated: true };
|
|
4108
4780
|
// Read-only callers persist the exact migration while they still hold the
|
|
4109
4781
|
// scope lock. Mutating callers use this same value for their final write.
|
|
@@ -4118,13 +4790,25 @@ function writeGovernorState(path, state) {
|
|
|
4118
4790
|
const parent = dirname(path);
|
|
4119
4791
|
let tempPath = null;
|
|
4120
4792
|
try {
|
|
4793
|
+
if (state?.version === GOVERNOR_STATE_VERSION) {
|
|
4794
|
+
if (!Number.isSafeInteger(state.revision) || state.revision === Number.MAX_SAFE_INTEGER) {
|
|
4795
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4796
|
+
}
|
|
4797
|
+
state.revision += 1;
|
|
4798
|
+
}
|
|
4799
|
+
const serialized = serializeGovernorState(state);
|
|
4800
|
+
if (state?.version === GOVERNOR_STATE_VERSION &&
|
|
4801
|
+
Buffer.byteLength(serialized) > GOVERNOR_MAX_LEDGER_BYTES) {
|
|
4802
|
+
return { ok: false, reason: "capacity" };
|
|
4803
|
+
}
|
|
4121
4804
|
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
4122
4805
|
chmodSync(parent, 0o700);
|
|
4123
4806
|
tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
4124
|
-
writeFileSync(tempPath,
|
|
4807
|
+
writeFileSync(tempPath, serialized, { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
4125
4808
|
chmodSync(tempPath, 0o600);
|
|
4809
|
+
// The private temporary inode already has mode 0600. Rename is the commit
|
|
4810
|
+
// point: no fallible operation may report failure after state replacement.
|
|
4126
4811
|
renameSync(tempPath, path);
|
|
4127
|
-
chmodSync(path, 0o600);
|
|
4128
4812
|
return { ok: true };
|
|
4129
4813
|
} catch {
|
|
4130
4814
|
if (tempPath !== null) {
|
|
@@ -4134,6 +4818,15 @@ function writeGovernorState(path, state) {
|
|
|
4134
4818
|
}
|
|
4135
4819
|
}
|
|
4136
4820
|
|
|
4821
|
+
function writeGovernorQuotaState(path, state, nowMs) {
|
|
4822
|
+
const normalized = normalizeGovernorState(state, nowMs, { prune: false });
|
|
4823
|
+
if (!normalized) return { ok: false, reason: "corrupt" };
|
|
4824
|
+
if (!governorAccountingSafe(normalized, nowMs)) {
|
|
4825
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
4826
|
+
}
|
|
4827
|
+
return writeGovernorState(path, normalized);
|
|
4828
|
+
}
|
|
4829
|
+
|
|
4137
4830
|
function lockOwner(path) {
|
|
4138
4831
|
try {
|
|
4139
4832
|
const owner = JSON.parse(readFileSync(path, "utf8"));
|
|
@@ -4443,9 +5136,18 @@ function mutateGovernor(scope, nowMs, mutate) {
|
|
|
4443
5136
|
const loaded = readGovernorState(scope.path, at, { persistMigration: false });
|
|
4444
5137
|
if (!loaded.ok) return loaded;
|
|
4445
5138
|
const state = loaded.value;
|
|
5139
|
+
if (!governorAccountingSafe(state, at)) return { ok: false, reason: "accounting-overflow" };
|
|
5140
|
+
const oldBootReconciled = reconcileBootDebt(state, at);
|
|
5141
|
+
if (!oldBootReconciled.ok) return oldBootReconciled;
|
|
5142
|
+
const compacted = compactGovernorReceipts(state, at);
|
|
5143
|
+
if (!compacted.ok) return compacted;
|
|
5144
|
+
const bootReconciled = reconcileBootDebt(state, at);
|
|
5145
|
+
if (!bootReconciled.ok) return bootReconciled;
|
|
4446
5146
|
// A mutator can reject without writing. Keep an untouched migration so
|
|
4447
5147
|
// that even that path upgrades the file once before releasing the lock.
|
|
4448
|
-
const migration = loaded.migrated
|
|
5148
|
+
const migration = loaded.migrated || compacted.changed ||
|
|
5149
|
+
oldBootReconciled.changed || bootReconciled.changed
|
|
5150
|
+
? structuredClone(state) : null;
|
|
4449
5151
|
const returnAfterMigration = (result) => {
|
|
4450
5152
|
if (migration === null) return result;
|
|
4451
5153
|
const written = writeGovernorState(scope.path, migration);
|
|
@@ -4453,6 +5155,7 @@ function mutateGovernor(scope, nowMs, mutate) {
|
|
|
4453
5155
|
};
|
|
4454
5156
|
const value = mutate(state, at);
|
|
4455
5157
|
if (value?.ok === false && value.write !== true) return returnAfterMigration(value);
|
|
5158
|
+
if (!governorAccountingSafe(state, at)) return returnAfterMigration({ ok: false, reason: "accounting-overflow" });
|
|
4456
5159
|
const normalized = normalizeGovernorState(state, at);
|
|
4457
5160
|
if (!normalized) return returnAfterMigration({ ok: false, reason: "corrupt" });
|
|
4458
5161
|
const written = writeGovernorState(scope.path, normalized);
|
|
@@ -4481,7 +5184,7 @@ const GOVERNOR_MAX_ATOMIC_COST = Math.max(...Object.values(OPERATION_COSTS)
|
|
|
4481
5184
|
function returnPacingCredit(state, resource, unusedCost, nowMs) {
|
|
4482
5185
|
const budget = state.budgets[resource];
|
|
4483
5186
|
if (!budget || !(unusedCost > 0)) return;
|
|
4484
|
-
const decision =
|
|
5187
|
+
const decision = governorResourceDecision(state, resource, nowMs, 0);
|
|
4485
5188
|
const callsPerMs = decision?.callsPerMs;
|
|
4486
5189
|
if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) return;
|
|
4487
5190
|
const credit = Math.min(unusedCost, GOVERNOR_MAX_ATOMIC_COST) / callsPerMs;
|
|
@@ -4544,6 +5247,10 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
4544
5247
|
leases: state.leases,
|
|
4545
5248
|
budgets: state.budgets,
|
|
4546
5249
|
reservations: Object.entries(state.reservations).map(([id, reservation]) => ({ id, ...reservation })),
|
|
5250
|
+
debtCosts: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
5251
|
+
state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
|
|
5252
|
+
reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs),
|
|
5253
|
+
])),
|
|
4547
5254
|
lanes,
|
|
4548
5255
|
cursors,
|
|
4549
5256
|
nowMs,
|
|
@@ -4572,6 +5279,7 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
4572
5279
|
startedAt: null,
|
|
4573
5280
|
completedAt: null,
|
|
4574
5281
|
outcome: null,
|
|
5282
|
+
receipt: null,
|
|
4575
5283
|
};
|
|
4576
5284
|
reservationCount += 1;
|
|
4577
5285
|
delete state.intents[grant.intentId];
|
|
@@ -4676,6 +5384,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4676
5384
|
dueAtCoreReset,
|
|
4677
5385
|
);
|
|
4678
5386
|
if (nextAt > at) return { value: { status: "waiting", nextAt } };
|
|
5387
|
+
if (resource === "graphql" && state.controlReceipts.graphql) {
|
|
5388
|
+
// The old claim's lease has expired. Keep an issued child charged as
|
|
5389
|
+
// debt, and fence the old capability so a new observer can make progress.
|
|
5390
|
+
if (!retireControlReceiptToDebt(state, "graphql", at)) {
|
|
5391
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5392
|
+
}
|
|
5393
|
+
}
|
|
4679
5394
|
const nonce = randomUUID();
|
|
4680
5395
|
const startedReservationIds = Object.entries(state.reservations)
|
|
4681
5396
|
// An expired owner can no longer settle its request. Keep that uncertain
|
|
@@ -4686,6 +5401,36 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4686
5401
|
state.leases[reservation.leaseId]?.expiresAt > at &&
|
|
4687
5402
|
reservationCost(reservation, resource, state.leases, at) > 0)
|
|
4688
5403
|
.map(([id]) => id);
|
|
5404
|
+
const quiescentGroups = Object.values(state.debtGroups).filter((group) =>
|
|
5405
|
+
group.quiescent && group.barrierAt < at &&
|
|
5406
|
+
(group.units[resource] > 0 || group.counts[resource] > 0));
|
|
5407
|
+
const barrierAt = Math.max(0, ...quiescentGroups.map((group) => group.barrierAt));
|
|
5408
|
+
const debtUnits = quiescentGroups.reduce((sum, group) => sum + group.units[resource], 0);
|
|
5409
|
+
const debtCount = quiescentGroups.reduce((sum, group) => sum + group.counts[resource], 0);
|
|
5410
|
+
if (!Number.isSafeInteger(debtUnits) || !Number.isSafeInteger(debtCount)) {
|
|
5411
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5412
|
+
}
|
|
5413
|
+
const debtSnapshot = {
|
|
5414
|
+
units: debtUnits,
|
|
5415
|
+
count: debtCount,
|
|
5416
|
+
barrierAt,
|
|
5417
|
+
revision: state.debt[resource].revision,
|
|
5418
|
+
};
|
|
5419
|
+
if (resource === "graphql") {
|
|
5420
|
+
state.controlReceipts.graphql = {
|
|
5421
|
+
leaseId, intentId: nonce, costs: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS },
|
|
5422
|
+
actualCosts: null, accountedCosts: { core: 0, graphql: 0 },
|
|
5423
|
+
notBefore: at, status: "started",
|
|
5424
|
+
epochs: { core: null, graphql: state.epochs.graphql },
|
|
5425
|
+
startedAt: at, completedAt: null, outcome: null,
|
|
5426
|
+
receipt: {
|
|
5427
|
+
scopeHash: scope.hash, ownerNonce: governorId(), generation: governorId(),
|
|
5428
|
+
ownerPid: process.pid, ownerBirth: coordinationProcessBirth(), bootId: coordinationBootId(),
|
|
5429
|
+
deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
|
|
5430
|
+
allowance: { core: 0, graphql: GRAPHQL_OBSERVER_POINTS }, dispatches: [],
|
|
5431
|
+
},
|
|
5432
|
+
};
|
|
5433
|
+
}
|
|
4689
5434
|
state.probeClaims[resource] = {
|
|
4690
5435
|
ownerLeaseId: leaseId,
|
|
4691
5436
|
nonce,
|
|
@@ -4693,6 +5438,7 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4693
5438
|
nextAt: at,
|
|
4694
5439
|
claimAt: at,
|
|
4695
5440
|
startedReservationIds,
|
|
5441
|
+
debtSnapshot,
|
|
4696
5442
|
};
|
|
4697
5443
|
state.observers[resource].outcome = "waiting";
|
|
4698
5444
|
state.observers[resource].at = at;
|
|
@@ -4704,6 +5450,13 @@ function claimProbe(scope, leaseId, nowMs, resource) {
|
|
|
4704
5450
|
leaseUntil: state.probeClaims[resource].leaseUntil,
|
|
4705
5451
|
startedReservationIds,
|
|
4706
5452
|
coreEtag: state.observers[resource].etag,
|
|
5453
|
+
...(resource === "graphql" ? { receiptCapability: {
|
|
5454
|
+
reservationId: "control:graphql", controlResource: "graphql", leaseId,
|
|
5455
|
+
scopeHash: scope.hash,
|
|
5456
|
+
ownerNonce: state.controlReceipts.graphql.receipt.ownerNonce,
|
|
5457
|
+
generation: state.controlReceipts.graphql.receipt.generation,
|
|
5458
|
+
deadline: state.controlReceipts.graphql.receipt.deadline,
|
|
5459
|
+
} } : {}),
|
|
4707
5460
|
} };
|
|
4708
5461
|
});
|
|
4709
5462
|
}
|
|
@@ -4785,6 +5538,34 @@ function budgetFromObservation(raw, previous, nowMs, {
|
|
|
4785
5538
|
return { status: "accepted", budget: next, epochChanged };
|
|
4786
5539
|
}
|
|
4787
5540
|
|
|
5541
|
+
function retireClaimedQuiescentDebt(state, resource, claim) {
|
|
5542
|
+
const snapshot = claim.debtSnapshot;
|
|
5543
|
+
if (snapshot.units === 0 && snapshot.count === 0) return true;
|
|
5544
|
+
if (claim.claimAt <= snapshot.barrierAt) return true;
|
|
5545
|
+
const eligible = Object.entries(state.debtGroups).filter(([, group]) =>
|
|
5546
|
+
group.quiescent && group.barrierAt < claim.claimAt &&
|
|
5547
|
+
group.barrierRevisions[resource] <= snapshot.revision &&
|
|
5548
|
+
(group.units[resource] > 0 || group.counts[resource] > 0));
|
|
5549
|
+
const units = eligible.reduce((total, [, group]) => total + group.units[resource], 0);
|
|
5550
|
+
const count = eligible.reduce((total, [, group]) => total + group.counts[resource], 0);
|
|
5551
|
+
const debt = state.debt[resource];
|
|
5552
|
+
if (!Number.isSafeInteger(units) || !Number.isSafeInteger(count) ||
|
|
5553
|
+
units !== snapshot.units || count !== snapshot.count ||
|
|
5554
|
+
debt.quiescentUnits < units || debt.quiescentCount < count ||
|
|
5555
|
+
!Number.isSafeInteger(debt.revision + 1)) return false;
|
|
5556
|
+
debt.quiescentUnits -= units;
|
|
5557
|
+
debt.quiescentCount -= count;
|
|
5558
|
+
debt.revision += 1;
|
|
5559
|
+
for (const [id, group] of eligible) {
|
|
5560
|
+
group.units[resource] = 0;
|
|
5561
|
+
group.counts[resource] = 0;
|
|
5562
|
+
if (RATE_RESOURCES.every((name) => group.units[name] === 0 && group.counts[name] === 0)) {
|
|
5563
|
+
delete state.debtGroups[id];
|
|
5564
|
+
}
|
|
5565
|
+
}
|
|
5566
|
+
return true;
|
|
5567
|
+
}
|
|
5568
|
+
|
|
4788
5569
|
function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
|
|
4789
5570
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
4790
5571
|
const claim = state.probeClaims[resource];
|
|
@@ -4833,6 +5614,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
|
|
|
4833
5614
|
}
|
|
4834
5615
|
}
|
|
4835
5616
|
if (changedResources.length === 0) return { ok: false, reason: "corrupt" };
|
|
5617
|
+
if (!retireClaimedQuiescentDebt(state, resource, claim)) {
|
|
5618
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5619
|
+
}
|
|
4836
5620
|
const completedBeforeClaim = Object.entries(state.reservations)
|
|
4837
5621
|
.filter(([, reservation]) =>
|
|
4838
5622
|
reservation.status === "completed" && reservation.completedAt < claim.claimAt);
|
|
@@ -4884,6 +5668,9 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
|
|
|
4884
5668
|
state.budgets = nextBudgets;
|
|
4885
5669
|
state.epochs = nextEpochs;
|
|
4886
5670
|
state.probeClaims[resource] = null;
|
|
5671
|
+
if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
|
|
5672
|
+
state.controlReceipts.graphql = null;
|
|
5673
|
+
}
|
|
4887
5674
|
{
|
|
4888
5675
|
const published = nextBudgets[resource];
|
|
4889
5676
|
state.observers[resource] = {
|
|
@@ -4944,6 +5731,15 @@ function failProbeClaim(scope, leaseId, nonce, nowMs, resource) {
|
|
|
4944
5731
|
// also marked core failed would erase authority core had legitimately
|
|
4945
5732
|
// established, and stall a lane with nothing wrong with it.
|
|
4946
5733
|
state.probeClaims[resource] = null;
|
|
5734
|
+
if (resource === "graphql" && state.controlReceipts.graphql?.intentId === nonce) {
|
|
5735
|
+
const control = state.controlReceipts.graphql;
|
|
5736
|
+
if (control.receipt.dispatches.length === 0) state.controlReceipts.graphql = null;
|
|
5737
|
+
else if (control.receipt.dispatches.every((dispatch) => dispatch.terminalAt !== null)) {
|
|
5738
|
+
if (!retireControlReceiptToDebt(state, "graphql", at)) {
|
|
5739
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
5740
|
+
}
|
|
5741
|
+
}
|
|
5742
|
+
}
|
|
4947
5743
|
state.observers[resource] = {
|
|
4948
5744
|
...state.observers[resource],
|
|
4949
5745
|
outcome: "failed",
|
|
@@ -5118,7 +5914,8 @@ function startReservation(scope, reservationId, nowMs) {
|
|
|
5118
5914
|
}
|
|
5119
5915
|
const charged = Object.values(state.reservations).reduce(
|
|
5120
5916
|
(total, item) => total + reservationCost(item, resource, state.leases, at),
|
|
5121
|
-
|
|
5917
|
+
state.debt[resource].unresolvedUnits + state.debt[resource].quiescentUnits +
|
|
5918
|
+
reservationCost(state.controlReceipts[resource], resource, state.leases, at),
|
|
5122
5919
|
);
|
|
5123
5920
|
if (budget.remaining - resourceReserve(budget.limit) - charged < 0) {
|
|
5124
5921
|
reservation.status = "cancelled";
|
|
@@ -5127,43 +5924,204 @@ function startReservation(scope, reservationId, nowMs) {
|
|
|
5127
5924
|
}
|
|
5128
5925
|
reservation.status = "started";
|
|
5129
5926
|
reservation.startedAt = at;
|
|
5130
|
-
|
|
5927
|
+
const generation = state.ownerGenerations[reservation.leaseId] ?? governorId();
|
|
5928
|
+
state.ownerGenerations[reservation.leaseId] = generation;
|
|
5929
|
+
reservation.receipt = {
|
|
5930
|
+
scopeHash: scope.hash,
|
|
5931
|
+
ownerNonce: generation,
|
|
5932
|
+
generation,
|
|
5933
|
+
ownerPid: process.pid,
|
|
5934
|
+
ownerBirth: coordinationProcessBirth(),
|
|
5935
|
+
bootId: coordinationBootId(),
|
|
5936
|
+
deadline: at + ACQUISITION_STARTED_DEADLINE_MS,
|
|
5937
|
+
allowance: { ...reservation.costs },
|
|
5938
|
+
dispatches: [],
|
|
5939
|
+
};
|
|
5940
|
+
return { value: { status: "started", reservationId,
|
|
5941
|
+
receiptCapability: receiptCapabilityFromReservation(reservationId, reservation) } };
|
|
5131
5942
|
});
|
|
5132
5943
|
}
|
|
5133
5944
|
|
|
5134
|
-
function
|
|
5945
|
+
function mutateReceiptDispatch(scope, capability, nowMs, mutate) {
|
|
5946
|
+
if (!capability || capability.scopeHash !== scope?.hash ||
|
|
5947
|
+
!validGovernorId(capability.ownerNonce) || !validGovernorId(capability.generation) ||
|
|
5948
|
+
!validGovernorId(capability.leaseId) ||
|
|
5949
|
+
typeof capability.reservationId !== "string") return { ok: false, reason: "receipt-invalid" };
|
|
5135
5950
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
5136
|
-
const
|
|
5137
|
-
if (
|
|
5138
|
-
return { ok: false, reason: "
|
|
5139
|
-
}
|
|
5140
|
-
const
|
|
5141
|
-
?
|
|
5142
|
-
|
|
5143
|
-
|
|
5144
|
-
|
|
5145
|
-
|
|
5146
|
-
|
|
5147
|
-
|
|
5148
|
-
|
|
5149
|
-
|
|
5150
|
-
|
|
5151
|
-
|
|
5152
|
-
|
|
5153
|
-
|
|
5154
|
-
|
|
5155
|
-
|
|
5156
|
-
|
|
5157
|
-
|
|
5158
|
-
|
|
5159
|
-
|
|
5160
|
-
|
|
5951
|
+
const controlResource = capability.controlResource;
|
|
5952
|
+
if (controlResource !== undefined && !RATE_RESOURCES.includes(controlResource)) {
|
|
5953
|
+
return { ok: false, reason: "receipt-invalid" };
|
|
5954
|
+
}
|
|
5955
|
+
const reservation = controlResource
|
|
5956
|
+
? state.controlReceipts[controlResource] : state.reservations[capability.reservationId];
|
|
5957
|
+
const receipt = reservation?.receipt;
|
|
5958
|
+
if (!receipt || receipt.scopeHash !== capability.scopeHash ||
|
|
5959
|
+
receipt.ownerNonce !== capability.ownerNonce || receipt.generation !== capability.generation ||
|
|
5960
|
+
reservation.leaseId !== capability.leaseId || receipt.deadline !== capability.deadline) {
|
|
5961
|
+
return { ok: false, reason: "compacted" };
|
|
5962
|
+
}
|
|
5963
|
+
return mutate(state, reservation, receipt, at);
|
|
5964
|
+
});
|
|
5965
|
+
}
|
|
5966
|
+
|
|
5967
|
+
function issueGovernorDispatch(scope, capability, operation, nowMs) {
|
|
5968
|
+
const costs = operationCost(operation);
|
|
5969
|
+
if (!costs) return { ok: false, reason: "undeclared" };
|
|
5970
|
+
if ([...terminalizationBacklog.keys()].filter((key) =>
|
|
5971
|
+
key.startsWith(`${scope?.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
|
|
5972
|
+
return { ok: false, reason: "completion-capacity" };
|
|
5973
|
+
}
|
|
5974
|
+
return mutateReceiptDispatch(scope, capability, nowMs, (state, reservation, receipt, at) => {
|
|
5975
|
+
if (reservation.status !== "started" || receipt.deadline <= at ||
|
|
5976
|
+
(!capability.controlResource && (
|
|
5977
|
+
state.ownerGenerations[reservation.leaseId] !== receipt.generation ||
|
|
5978
|
+
state.leases[reservation.leaseId]?.expiresAt <= at ||
|
|
5979
|
+
!state.leases[reservation.leaseId]))) return { ok: false, reason: "receipt-expired" };
|
|
5980
|
+
const issued = { core: 0, graphql: 0 };
|
|
5981
|
+
for (const dispatch of receipt.dispatches) {
|
|
5982
|
+
for (const resource of RATE_RESOURCES) issued[resource] += dispatch.costs[resource];
|
|
5983
|
+
}
|
|
5984
|
+
if (RATE_RESOURCES.some((resource) =>
|
|
5985
|
+
!Number.isSafeInteger(issued[resource] + costs[resource]) ||
|
|
5986
|
+
issued[resource] + costs[resource] > receipt.allowance[resource])) {
|
|
5987
|
+
return { ok: false, reason: "receipt-allowance" };
|
|
5988
|
+
}
|
|
5989
|
+
const sequence = receipt.dispatches.length + 1;
|
|
5990
|
+
if (sequence > 32) return { ok: false, reason: "receipt-allowance" };
|
|
5991
|
+
receipt.dispatches.push({ sequence, operation, costs: { ...costs },
|
|
5992
|
+
issuedAt: at, terminalAt: null, childPid: null, childBirth: null, neverStarted: false });
|
|
5993
|
+
return { value: { sequence, deadline: receipt.deadline } };
|
|
5994
|
+
});
|
|
5995
|
+
}
|
|
5996
|
+
|
|
5997
|
+
function markGovernorDispatchChild(scope, capability, sequence, pid, nowMs) {
|
|
5998
|
+
if (!Number.isSafeInteger(pid) || pid < 1) return { ok: false, reason: "child-unknown" };
|
|
5999
|
+
return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
|
|
6000
|
+
(_state, _reservation, receipt) => {
|
|
6001
|
+
const dispatch = receipt.dispatches[sequence - 1];
|
|
6002
|
+
if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
|
|
6003
|
+
dispatch.childPid = pid;
|
|
6004
|
+
dispatch.childBirth = coordinationProcessBirth(pid);
|
|
6005
|
+
return { value: { status: "recorded" } };
|
|
6006
|
+
});
|
|
6007
|
+
}
|
|
6008
|
+
|
|
6009
|
+
function terminalGovernorDispatch(scope, capability, sequence, nowMs, { neverStarted = false } = {}) {
|
|
6010
|
+
return mutateReceiptDispatch({ ...scope, identityProvider: null }, capability, nowMs,
|
|
6011
|
+
(_state, _reservation, receipt, at) => {
|
|
6012
|
+
const dispatch = receipt.dispatches[sequence - 1];
|
|
6013
|
+
if (!dispatch || dispatch.sequence !== sequence) return { ok: false, reason: "compacted" };
|
|
6014
|
+
if (neverStarted && dispatch.childPid !== null) return { ok: false, reason: "child-started" };
|
|
6015
|
+
if (dispatch.terminalAt === null) {
|
|
6016
|
+
dispatch.terminalAt = at;
|
|
6017
|
+
dispatch.neverStarted = neverStarted;
|
|
5161
6018
|
}
|
|
5162
|
-
|
|
5163
|
-
|
|
6019
|
+
return { value: { status: "terminal" } };
|
|
6020
|
+
});
|
|
6021
|
+
}
|
|
6022
|
+
|
|
6023
|
+
const activeOwnerDispatches = new Map();
|
|
6024
|
+
function ownerDispatchKey(scope, capability) {
|
|
6025
|
+
return `${scope.hash}:${capability.generation}`;
|
|
6026
|
+
}
|
|
6027
|
+
|
|
6028
|
+
function acknowledgeSealedGeneration(scope, capability, sealedNonce, nowMs) {
|
|
6029
|
+
const key = ownerDispatchKey(scope, capability);
|
|
6030
|
+
if ((activeOwnerDispatches.get(key) ?? 0) !== 0) return { ok: false, reason: "children-active" };
|
|
6031
|
+
return mutateGovernor({ ...scope, identityProvider: null }, nowMs, (state, at) => {
|
|
6032
|
+
const group = state.debtGroups[capability.generation];
|
|
6033
|
+
if (!group) return { value: { status: "absent" } };
|
|
6034
|
+
if (group.nonce !== sealedNonce) return { ok: false, reason: "stale" };
|
|
6035
|
+
if (group.quiescent) return { value: { status: "acknowledged", nonce: group.nonce } };
|
|
6036
|
+
if (!group.ownerNonce || group.ownerNonce !== capability.ownerNonce ||
|
|
6037
|
+
!group.ownerBirth || !group.bootId ||
|
|
6038
|
+
group.ownerPid !== process.pid || group.ownerBirth !== coordinationProcessBirth() ||
|
|
6039
|
+
group.bootId !== coordinationBootId() ||
|
|
6040
|
+
Object.values(state.reservations).some((reservation) =>
|
|
6041
|
+
reservation.receipt?.generation === capability.generation)) {
|
|
6042
|
+
return { ok: false, reason: "quiescence-unverified" };
|
|
6043
|
+
}
|
|
6044
|
+
if (!makeDebtGroupQuiescent(state, group, at)) return { ok: false, reason: "accounting-overflow" };
|
|
6045
|
+
return { value: { status: "acknowledged", nonce: group.nonce } };
|
|
5164
6046
|
});
|
|
5165
6047
|
}
|
|
5166
6048
|
|
|
6049
|
+
function completeReservation(scope, reservationId, completion, nowMs) {
|
|
6050
|
+
const reservation = inspectGovernor({ ...scope, identityProvider: null }, nowMs);
|
|
6051
|
+
if (!reservation.ok) return reservation;
|
|
6052
|
+
const leaseId = reservation.value.reservations[reservationId]?.leaseId;
|
|
6053
|
+
if (!leaseId) return { ok: false, reason: "stale" };
|
|
6054
|
+
return settleReservationWithBudgetObservations(scope, leaseId, reservationId, {
|
|
6055
|
+
...completion,
|
|
6056
|
+
...(completion.actualCost ? { actualCosts: completion.actualCost } : {}),
|
|
6057
|
+
observations: [],
|
|
6058
|
+
}, nowMs);
|
|
6059
|
+
}
|
|
6060
|
+
|
|
6061
|
+
function terminalizeReservation(scope, capability, completion, nowMs) {
|
|
6062
|
+
if (!capability?.reservationId || !capability.leaseId) {
|
|
6063
|
+
return { ok: false, reason: "blocked", status: "blocked" };
|
|
6064
|
+
}
|
|
6065
|
+
const result = settleReservationWithBudgetObservations(
|
|
6066
|
+
{ ...scope, identityProvider: null }, capability.leaseId,
|
|
6067
|
+
capability.reservationId, { ...completion, capability }, nowMs,
|
|
6068
|
+
);
|
|
6069
|
+
if (result.ok) return { ok: true, value: { ...result.value, status: "settled" } };
|
|
6070
|
+
const status = ["busy", "unwritable"].includes(result.reason) ? "retryable"
|
|
6071
|
+
: result.reason === "stale" ? "compacted" : "blocked";
|
|
6072
|
+
return { ...result, status };
|
|
6073
|
+
}
|
|
6074
|
+
|
|
6075
|
+
const terminalizationBacklog = new Map();
|
|
6076
|
+
function terminalizationBacklogForResource(backlog, scopeHash, resource) {
|
|
6077
|
+
if (!scopeHash) return 0;
|
|
6078
|
+
let count = 0;
|
|
6079
|
+
for (const item of backlog.values()) {
|
|
6080
|
+
if (item.scope.hash !== scopeHash) continue;
|
|
6081
|
+
if (item.capability?.costs?.[resource] > 0 ||
|
|
6082
|
+
!RATE_RESOURCES.some((name) => item.capability?.costs?.[name] > 0)) count += 1;
|
|
6083
|
+
}
|
|
6084
|
+
return count;
|
|
6085
|
+
}
|
|
6086
|
+
function receiptCapabilityFromReservation(reservationId, reservation) {
|
|
6087
|
+
if (!reservation?.receipt || !reservation?.costs) return null;
|
|
6088
|
+
return {
|
|
6089
|
+
reservationId, leaseId: reservation.leaseId,
|
|
6090
|
+
scopeHash: reservation.receipt.scopeHash,
|
|
6091
|
+
ownerNonce: reservation.receipt.ownerNonce,
|
|
6092
|
+
generation: reservation.receipt.generation,
|
|
6093
|
+
deadline: reservation.receipt.deadline,
|
|
6094
|
+
costs: { ...reservation.costs },
|
|
6095
|
+
};
|
|
6096
|
+
}
|
|
6097
|
+
function queueTerminalization(scope, capability, completion, now = Date.now) {
|
|
6098
|
+
const frozen = { ...scope, identityProvider: null };
|
|
6099
|
+
const key = `${scope.hash}:${capability?.reservationId ?? "invalid"}`;
|
|
6100
|
+
const first = terminalizeReservation(frozen, capability, completion, now());
|
|
6101
|
+
if (first.ok || first.status !== "retryable") return first;
|
|
6102
|
+
if (terminalizationBacklog.has(key)) return first;
|
|
6103
|
+
if ([...terminalizationBacklog.keys()].filter((existing) =>
|
|
6104
|
+
existing.startsWith(`${scope.hash}:`)).length >= GOVERNOR_MAX_RESERVATIONS) {
|
|
6105
|
+
return { ok: false, reason: "completion-capacity", status: "blocked" };
|
|
6106
|
+
}
|
|
6107
|
+
const item = { scope: frozen, capability, completion, now, attempt: 0, timer: null };
|
|
6108
|
+
const retry = () => {
|
|
6109
|
+
const result = terminalizeReservation(item.scope, item.capability, item.completion, item.now());
|
|
6110
|
+
if (result.ok || result.status !== "retryable") {
|
|
6111
|
+
terminalizationBacklog.delete(key);
|
|
6112
|
+
return;
|
|
6113
|
+
}
|
|
6114
|
+
const delays = [1000, 2000, 4000, 5000];
|
|
6115
|
+
const delay = delays[Math.min(item.attempt++, delays.length - 1)];
|
|
6116
|
+
item.timer = setTimeout(retry, delay + Math.floor(Math.random() * 100));
|
|
6117
|
+
item.timer.unref?.();
|
|
6118
|
+
};
|
|
6119
|
+
terminalizationBacklog.set(key, item);
|
|
6120
|
+
item.timer = setTimeout(retry, 1000);
|
|
6121
|
+
item.timer.unref?.();
|
|
6122
|
+
return first;
|
|
6123
|
+
}
|
|
6124
|
+
|
|
5167
6125
|
function settleReservationWithBudgetObservations(
|
|
5168
6126
|
scope,
|
|
5169
6127
|
leaseId,
|
|
@@ -5173,13 +6131,33 @@ function settleReservationWithBudgetObservations(
|
|
|
5173
6131
|
) {
|
|
5174
6132
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
5175
6133
|
const reservation = state.reservations[reservationId];
|
|
5176
|
-
|
|
6134
|
+
if (reservation?.status === "completed" && reservation.leaseId === leaseId &&
|
|
6135
|
+
(!completion?.capability ||
|
|
6136
|
+
completion.capability.ownerNonce === reservation.receipt?.ownerNonce &&
|
|
6137
|
+
completion.capability.generation === reservation.receipt?.generation)) {
|
|
6138
|
+
return { value: { status: "completed", actualCosts: reservation.actualCosts,
|
|
6139
|
+
accountedCosts: reservation.accountedCosts } };
|
|
6140
|
+
}
|
|
5177
6141
|
if (
|
|
5178
|
-
!
|
|
6142
|
+
!reservation ||
|
|
5179
6143
|
reservation.leaseId !== leaseId || reservation.status !== "started" ||
|
|
5180
6144
|
!GOVERNOR_OUTCOMES.has(completion?.outcome)
|
|
5181
6145
|
) return { ok: false, reason: "stale" };
|
|
5182
|
-
|
|
6146
|
+
if (completion.capability && (completion.capability.scopeHash !== scope.hash ||
|
|
6147
|
+
completion.capability.reservationId !== reservationId ||
|
|
6148
|
+
completion.capability.leaseId !== leaseId ||
|
|
6149
|
+
completion.capability.ownerNonce !== reservation.receipt?.ownerNonce ||
|
|
6150
|
+
completion.capability.generation !== reservation.receipt?.generation ||
|
|
6151
|
+
completion.capability.deadline !== reservation.receipt?.deadline)) {
|
|
6152
|
+
return { ok: false, reason: "receipt-invalid" };
|
|
6153
|
+
}
|
|
6154
|
+
const neverIssued = completion.capability && (
|
|
6155
|
+
completion.neverStarted === true && reservation.receipt.dispatches.length === 0 ||
|
|
6156
|
+
reservation.receipt.dispatches.length > 0 &&
|
|
6157
|
+
reservation.receipt.dispatches.every((dispatch) => dispatch.neverStarted));
|
|
6158
|
+
const outcome = neverIssued ? "measured-success" : completion.outcome;
|
|
6159
|
+
const measured = neverIssued ? { core: 0, graphql: 0 }
|
|
6160
|
+
: outcome === "measured-success"
|
|
5183
6161
|
? exactResourceCosts(completion.actualCosts ?? completion.actualCost)
|
|
5184
6162
|
: reservation.costs;
|
|
5185
6163
|
if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
|
|
@@ -5231,17 +6209,17 @@ function settleReservationWithBudgetObservations(
|
|
|
5231
6209
|
}
|
|
5232
6210
|
reservation.status = "completed";
|
|
5233
6211
|
reservation.completedAt = at;
|
|
5234
|
-
reservation.outcome =
|
|
6212
|
+
reservation.outcome = outcome;
|
|
5235
6213
|
reservation.actualCosts = { ...measured };
|
|
5236
6214
|
reservation.accountedCosts = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
5237
6215
|
resource,
|
|
5238
|
-
|
|
6216
|
+
outcome === "measured-success"
|
|
5239
6217
|
? Math.min(measured[resource], acceptedCosts[resource])
|
|
5240
6218
|
: 0,
|
|
5241
6219
|
]));
|
|
5242
6220
|
for (const resource of RATE_RESOURCES) {
|
|
5243
6221
|
const budget = state.budgets[resource];
|
|
5244
|
-
if (!budget ||
|
|
6222
|
+
if (!budget || outcome !== "measured-success") continue;
|
|
5245
6223
|
// accountedCosts removes the portion already present in an authoritative
|
|
5246
6224
|
// counter from the reservation residual. knownLocalUsed contains only
|
|
5247
6225
|
// that same portion when it arrived after the factor baseline. Their sum
|
|
@@ -5255,7 +6233,7 @@ function settleReservationWithBudgetObservations(
|
|
|
5255
6233
|
// loss proves nothing, so its worst case stays charged and its pacing stays
|
|
5256
6234
|
// spent -- refunding there would let a run of timeouts pace as though
|
|
5257
6235
|
// nothing had been sent.
|
|
5258
|
-
if (
|
|
6236
|
+
if (outcome === "measured-success") {
|
|
5259
6237
|
for (const resource of RATE_RESOURCES) {
|
|
5260
6238
|
returnPacingCredit(state, resource, reservation.costs[resource] - measured[resource], at);
|
|
5261
6239
|
}
|
|
@@ -5341,6 +6319,40 @@ function governorHealth(result, nowMs = Date.now()) {
|
|
|
5341
6319
|
};
|
|
5342
6320
|
}
|
|
5343
6321
|
|
|
6322
|
+
// Plain doctor must not call the identity coordinator: even its inspection
|
|
6323
|
+
// path may retire old registry attempts and persist the resulting registry.
|
|
6324
|
+
// A cached identity is useful only when it identifies one quota account for
|
|
6325
|
+
// the requested host. Multiple accounts cannot be resolved without probing.
|
|
6326
|
+
function doctorCachedQuotaScope(host, { root = identityRegistryRoot(), nowMs = Date.now() } = {}) {
|
|
6327
|
+
const registryPath = join(root, "registry.json");
|
|
6328
|
+
let registry;
|
|
6329
|
+
try {
|
|
6330
|
+
const file = lstatSync(registryPath);
|
|
6331
|
+
if (!file.isFile() || file.size > GOVERNOR_MAX_LEDGER_BYTES) return { ok: false, reason: "unsafe registry" };
|
|
6332
|
+
const parsed = JSON.parse(readFileSync(registryPath, "utf8"));
|
|
6333
|
+
registry = normalizeIdentityRegistry(parsed, parsed.version === 1 ? 1 : IDENTITY_VERSION);
|
|
6334
|
+
} catch (error) {
|
|
6335
|
+
return { ok: false, reason: error?.code === "ENOENT" ? "cached identity missing" : "cached registry unreadable" };
|
|
6336
|
+
}
|
|
6337
|
+
if (!registry) return { ok: false, reason: "cached registry corrupt" };
|
|
6338
|
+
const quotas = new Set(Object.values(registry.identities)
|
|
6339
|
+
.filter((identity) => identity.host === host)
|
|
6340
|
+
.map((identity) => identity.quotaKey));
|
|
6341
|
+
if (quotas.size === 0) return { ok: false, reason: "cached identity missing" };
|
|
6342
|
+
if (quotas.size !== 1) return { ok: false, reason: "ambiguous cached identities" };
|
|
6343
|
+
const quotaKey = quotas.values().next().value;
|
|
6344
|
+
const path = join(root, `quota-${quotaKey}.json`);
|
|
6345
|
+
try {
|
|
6346
|
+
if (!lstatSync(path).isFile()) return { ok: false, reason: "unsafe quota path", path };
|
|
6347
|
+
} catch (error) {
|
|
6348
|
+
return { ok: false, reason: error?.code === "ENOENT" ? "cached quota missing" : "cached quota unreadable", path };
|
|
6349
|
+
}
|
|
6350
|
+
const result = readGovernorState(path, nowMs, { persistMigration: false });
|
|
6351
|
+
return result.ok && !result.missing
|
|
6352
|
+
? { ok: true, result, path, quotaKey }
|
|
6353
|
+
: { ok: false, reason: result.missing ? "cached quota missing" : result.reason, path };
|
|
6354
|
+
}
|
|
6355
|
+
|
|
5344
6356
|
async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
|
|
5345
6357
|
let result = run();
|
|
5346
6358
|
while (!result.ok && result.reason === "busy" && !signal?.aborted && now() < deadline) {
|
|
@@ -5470,7 +6482,8 @@ async function refreshResourceBudget(scope, leaseId, signal, resource, {
|
|
|
5470
6482
|
let budgets;
|
|
5471
6483
|
try {
|
|
5472
6484
|
try { onObserverCall?.(resource); } catch { /* diagnostics cannot block a budget observer */ }
|
|
5473
|
-
budgets = await requestIdentityStorage.run(
|
|
6485
|
+
budgets = await requestIdentityStorage.run({ ...scope,
|
|
6486
|
+
controlReceiptCapability: claim.value.receiptCapability }, () => readBudgets(signal, scope.host, {
|
|
5474
6487
|
resources: [resource],
|
|
5475
6488
|
coreEtag,
|
|
5476
6489
|
renewClaim: async () => {
|
|
@@ -6074,7 +7087,7 @@ const SSH_RECONNECT_STEPS_MS = Object.freeze([1_000, 2_000, 4_000, 8_000, 16_000
|
|
|
6074
7087
|
const SSH_STDERR_MAX_BYTES = 4_096;
|
|
6075
7088
|
const SSH_ALIAS_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,254}$/;
|
|
6076
7089
|
const COLLECTOR_HOLD_REASONS = new Set([
|
|
6077
|
-
"observer", "primary", "secondary", "disconnected", "coordination", "shared-wait", "cache-only",
|
|
7090
|
+
"observer", "primary", "secondary", "disconnected", "app-auth", "coordination", "shared-wait", "cache-only",
|
|
6078
7091
|
]);
|
|
6079
7092
|
const COLLECTOR_ROW_KEYS = Object.freeze({
|
|
6080
7093
|
actions: ["databaseId", "displayTitle", "workflowName", "number", "headBranch", "status", "conclusion",
|
|
@@ -6592,7 +7605,7 @@ function persistGitHubAppIdentity(root, identity, now = Date.now()) {
|
|
|
6592
7605
|
const initialized = withGovernorLock(scope, () => {
|
|
6593
7606
|
const loaded = readIdentityQuotaState(state, scope, now);
|
|
6594
7607
|
if (!loaded.ok) return loaded;
|
|
6595
|
-
return loaded.missing ?
|
|
7608
|
+
return loaded.missing ? writeGovernorQuotaState(scope.path, loaded.value, now) : { ok: true };
|
|
6596
7609
|
});
|
|
6597
7610
|
if (!initialized.ok) return initialized;
|
|
6598
7611
|
state.identities[identity.credentialKey] = { host: identity.host, kind: identity.kind,
|
|
@@ -8403,7 +9416,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
|
|
|
8403
9416
|
if (!["changed", "unchanged"].includes(transition.kind)) {
|
|
8404
9417
|
const error = new Error("collector received unusable data");
|
|
8405
9418
|
error.requestMetrics = acquisitionRequestMetrics(result);
|
|
8406
|
-
error.uncertainReceipts = result.uncertainReceipts ?? [];
|
|
8407
9419
|
throw error;
|
|
8408
9420
|
}
|
|
8409
9421
|
const rows = transition.kind === "changed" ? transition.data : previous?.rows;
|
|
@@ -8435,7 +9447,6 @@ function collectorPublicationFromResult(resource, result, previous, demand, comp
|
|
|
8435
9447
|
hold: null,
|
|
8436
9448
|
capabilities: result.capabilities ?? previous?.capabilities ?? {},
|
|
8437
9449
|
requestMetrics: acquisitionRequestMetrics(result),
|
|
8438
|
-
uncertainReceipts: result.uncertainReceipts ?? [],
|
|
8439
9450
|
repositoryIdentity: result.repositoryIdentity,
|
|
8440
9451
|
meta,
|
|
8441
9452
|
securityNotes: resource === "security"
|
|
@@ -8616,6 +9627,7 @@ function createCollectorAcquisitionRuntime({
|
|
|
8616
9627
|
},
|
|
8617
9628
|
});
|
|
8618
9629
|
const reservationId = admitted.reservationId;
|
|
9630
|
+
const transportScope = { ...context.scope, receiptCapability: admitted.receiptCapability };
|
|
8619
9631
|
const governor = inspectGovernor(context.scope, now());
|
|
8620
9632
|
const started = await markStarted({
|
|
8621
9633
|
reservationId,
|
|
@@ -8623,8 +9635,9 @@ function createCollectorAcquisitionRuntime({
|
|
|
8623
9635
|
epochs: governor.ok ? governor.value.epochs : {},
|
|
8624
9636
|
});
|
|
8625
9637
|
if (!started.ok || started.value?.status !== "started") {
|
|
8626
|
-
|
|
8627
|
-
{ outcome: "rejected" }, now
|
|
9638
|
+
const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
|
|
9639
|
+
{ outcome: "rejected", neverStarted: true }, now);
|
|
9640
|
+
if (terminal.status === "blocked") throw new Error("collector receipt settlement blocked");
|
|
8628
9641
|
throw new Error("collector acquisition start failed");
|
|
8629
9642
|
}
|
|
8630
9643
|
const descriptor = tabForKey(resource);
|
|
@@ -8636,20 +9649,18 @@ function createCollectorAcquisitionRuntime({
|
|
|
8636
9649
|
signal,
|
|
8637
9650
|
AbortSignal.timeout(ACQUISITION_STARTED_DEADLINE_MS),
|
|
8638
9651
|
]);
|
|
8639
|
-
let result = await requestIdentityStorage.run(
|
|
9652
|
+
let result = await requestIdentityStorage.run(transportScope, () => descriptor.fetch({
|
|
8640
9653
|
signal: boundedSignal,
|
|
8641
9654
|
entities,
|
|
8642
9655
|
previousRaw: force ? null : snapshot?.raw ?? null,
|
|
8643
9656
|
force,
|
|
8644
9657
|
pages: demand.pages,
|
|
8645
|
-
governor: { scope:
|
|
9658
|
+
governor: { scope: transportScope, leaseId: context.leaseId },
|
|
8646
9659
|
...(securityPolicy ? { sources: securityPolicy.sources } : {}),
|
|
8647
9660
|
}));
|
|
8648
9661
|
if (securityPolicy) result = applyGitHubAppSecurityPolicy(result, securityPolicy);
|
|
8649
|
-
|
|
8650
|
-
|
|
8651
|
-
context.leaseId,
|
|
8652
|
-
reservationId,
|
|
9662
|
+
const terminal = queueTerminalization(
|
|
9663
|
+
context.scope, admitted.receiptCapability,
|
|
8653
9664
|
result?.measuredSuccess === false
|
|
8654
9665
|
? { outcome: "rejected", observations: result.observations ?? [] }
|
|
8655
9666
|
: {
|
|
@@ -8660,8 +9671,11 @@ function createCollectorAcquisitionRuntime({
|
|
|
8660
9671
|
},
|
|
8661
9672
|
observations: result.observations ?? [],
|
|
8662
9673
|
},
|
|
8663
|
-
now
|
|
9674
|
+
now,
|
|
8664
9675
|
);
|
|
9676
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
9677
|
+
throw new Error(`collector receipt settlement ${terminal.status}`);
|
|
9678
|
+
}
|
|
8665
9679
|
const completedAt = now();
|
|
8666
9680
|
const publication = collectorPublicationFromResult(
|
|
8667
9681
|
resource, result, snapshot, demand, completedAt, claim.state,
|
|
@@ -8677,10 +9691,11 @@ function createCollectorAcquisitionRuntime({
|
|
|
8677
9691
|
if (reconciliation !== null) publication.nextDueAt = completedAt + reconciliation;
|
|
8678
9692
|
return publication;
|
|
8679
9693
|
} catch (error) {
|
|
8680
|
-
|
|
8681
|
-
{
|
|
8682
|
-
|
|
8683
|
-
|
|
9694
|
+
const terminal = queueTerminalization(context.scope, admitted.receiptCapability,
|
|
9695
|
+
{ outcome: governorOutcomeForError(error), observations: error.budgetObservations ?? [] }, now);
|
|
9696
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
9697
|
+
error.receiptSettlement = terminal.status;
|
|
9698
|
+
}
|
|
8684
9699
|
throw error;
|
|
8685
9700
|
}
|
|
8686
9701
|
}
|
|
@@ -8794,6 +9809,9 @@ function createCollectorAcquisitionRuntime({
|
|
|
8794
9809
|
item.nextDueAt = snapshot.nextDueAt;
|
|
8795
9810
|
if (Number.isSafeInteger(snapshot.generation)) item.lastDeliveredGeneration = snapshot.generation;
|
|
8796
9811
|
if (Number.isFinite(snapshot.lastSuccessAt)) item.lastDeliveredSuccessAt = snapshot.lastSuccessAt;
|
|
9812
|
+
if (publication && item.requestedAt !== null && snapshot.lastSuccessAt >= item.requestedAt) {
|
|
9813
|
+
item.requestedAt = null;
|
|
9814
|
+
}
|
|
8797
9815
|
item.onSnapshot(snapshot);
|
|
8798
9816
|
if (!publication || !Number.isSafeInteger(snapshot.generation)) return true;
|
|
8799
9817
|
for (const waiter of item.invalidations) {
|
|
@@ -8809,11 +9827,17 @@ function createCollectorAcquisitionRuntime({
|
|
|
8809
9827
|
}
|
|
8810
9828
|
|
|
8811
9829
|
function startPoll(item, { manual = false, force = false } = {}) {
|
|
9830
|
+
if (closed || item.closed) return null;
|
|
9831
|
+
if (manual && !demandEnabled(item.demand)) item.requestedAt ??= now();
|
|
9832
|
+
if (!item.id) {
|
|
9833
|
+
if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
|
|
9834
|
+
return startInitialize(item);
|
|
9835
|
+
}
|
|
8812
9836
|
if (item.pollPromise) {
|
|
8813
9837
|
if (manual) item.queuedRefresh = { force: force || item.queuedRefresh?.force === true };
|
|
8814
9838
|
return item.pollPromise;
|
|
8815
9839
|
}
|
|
8816
|
-
if (
|
|
9840
|
+
if (!demandEnabled(item.demand) && item.requestedAt === null) return null;
|
|
8817
9841
|
item.pollPromise = poll(item, { manual, force }).finally(() => {
|
|
8818
9842
|
item.pollPromise = null;
|
|
8819
9843
|
if (item.promotionGeneration !== null) {
|
|
@@ -8832,7 +9856,7 @@ function createCollectorAcquisitionRuntime({
|
|
|
8832
9856
|
}
|
|
8833
9857
|
|
|
8834
9858
|
function schedule(item, delay, operation = () => startPoll(item)) {
|
|
8835
|
-
if (closed || item.closed || !demandEnabled(item.demand)) return;
|
|
9859
|
+
if (closed || item.closed || (!demandEnabled(item.demand) && item.requestedAt === null)) return;
|
|
8836
9860
|
if (item.timer !== null) clearTimeout_(item.timer);
|
|
8837
9861
|
item.timer = setTimeout_(() => {
|
|
8838
9862
|
item.timer = null;
|
|
@@ -8898,7 +9922,8 @@ function createCollectorAcquisitionRuntime({
|
|
|
8898
9922
|
() => engine.currentClaim(subscriptionId, claim), () => producer({
|
|
8899
9923
|
target: item.target,
|
|
8900
9924
|
resource: item.resource,
|
|
8901
|
-
demand: { ...ownership.value.demand, background: item.demand.background
|
|
9925
|
+
demand: { ...ownership.value.demand, background: item.demand.background,
|
|
9926
|
+
active: ownership.value.demand?.active || item.requestedAt !== null },
|
|
8902
9927
|
snapshot: ownership.value.snapshot,
|
|
8903
9928
|
claim: { ...claim, state: item },
|
|
8904
9929
|
provider: item.provider,
|
|
@@ -8919,12 +9944,11 @@ function createCollectorAcquisitionRuntime({
|
|
|
8919
9944
|
engine.refresh(subscriptionId, { claim, failure: {
|
|
8920
9945
|
hold: "disconnected",
|
|
8921
9946
|
requestMetrics: produced.requestMetrics ?? {},
|
|
8922
|
-
uncertainReceipts: produced.uncertainReceipts ?? [],
|
|
8923
9947
|
} });
|
|
8924
9948
|
engine.unsubscribe(subscriptionId);
|
|
8925
9949
|
item.id = null;
|
|
8926
9950
|
targetInitializations.delete(targetInitializationKey(item.target));
|
|
8927
|
-
item.onHold?.("
|
|
9951
|
+
item.onHold?.("app-auth");
|
|
8928
9952
|
retryIn = 1;
|
|
8929
9953
|
return;
|
|
8930
9954
|
}
|
|
@@ -8937,7 +9961,6 @@ function createCollectorAcquisitionRuntime({
|
|
|
8937
9961
|
engine.refresh(subscriptionId, { claim, failure: {
|
|
8938
9962
|
hold: "disconnected",
|
|
8939
9963
|
requestMetrics: produced.requestMetrics ?? {},
|
|
8940
|
-
uncertainReceipts: produced.uncertainReceipts ?? [],
|
|
8941
9964
|
} });
|
|
8942
9965
|
item.onHold?.("disconnected");
|
|
8943
9966
|
retryIn = item.demand.floorMs;
|
|
@@ -8960,10 +9983,12 @@ function createCollectorAcquisitionRuntime({
|
|
|
8960
9983
|
else if (claim) engine.refresh(subscriptionId, { claim, failure: {
|
|
8961
9984
|
hold,
|
|
8962
9985
|
requestMetrics: error?.requestMetrics ?? {},
|
|
8963
|
-
uncertainReceipts: error?.uncertainReceipts ?? [],
|
|
8964
9986
|
} });
|
|
8965
9987
|
else engine.setHold(subscriptionId, hold, { resource: item.resource, accessKey: item.identity.accessKey });
|
|
8966
|
-
item.onHold?.(
|
|
9988
|
+
item.onHold?.(item.provider?.definition?.type === "github-app" &&
|
|
9989
|
+
(error?.apiResponse?.status === 401 || error?.message === "GitHub App token unavailable" ||
|
|
9990
|
+
classify(error) === "auth-problem")
|
|
9991
|
+
? "app-auth" : hold);
|
|
8967
9992
|
try { onDiagnostic({ stage: "acquire", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
|
|
8968
9993
|
retryIn = error?.notStarted && Number.isFinite(error.retryAt)
|
|
8969
9994
|
? Math.max(50, error.retryAt - now()) : Math.min(1_000, item.demand.floorMs);
|
|
@@ -8979,13 +10004,16 @@ function createCollectorAcquisitionRuntime({
|
|
|
8979
10004
|
if (item.id && item.identity?.accessKey) {
|
|
8980
10005
|
engine.setHold(item.id, hold, { resource: item.resource, accessKey: item.identity.accessKey });
|
|
8981
10006
|
}
|
|
8982
|
-
item.onHold?.(
|
|
10007
|
+
item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
|
|
10008
|
+
error?.message === "GitHub App token unavailable" ? "app-auth" : hold);
|
|
8983
10009
|
try { onDiagnostic({ stage: "provider-refresh", reason: redact(shortErr(error)) }); } catch { /* isolated */ }
|
|
8984
10010
|
retryIn = Math.min(1_000, item.demand.floorMs);
|
|
8985
10011
|
} finally {
|
|
8986
10012
|
item.polling = false;
|
|
8987
10013
|
item.pollPhase = null;
|
|
8988
|
-
schedule(item, retryIn, item.id
|
|
10014
|
+
schedule(item, retryIn, item.id
|
|
10015
|
+
? () => startPoll(item, { manual: item.requestedAt !== null })
|
|
10016
|
+
: () => startInitialize(item));
|
|
8989
10017
|
}
|
|
8990
10018
|
}
|
|
8991
10019
|
|
|
@@ -9018,12 +10046,16 @@ function createCollectorAcquisitionRuntime({
|
|
|
9018
10046
|
if (subscription.value.snapshot && !accessReset) {
|
|
9019
10047
|
deliverCollectorSnapshot(item, subscription.value.snapshot, bindingRevision);
|
|
9020
10048
|
}
|
|
9021
|
-
if (demandEnabled(item.demand)) {
|
|
9022
|
-
|
|
10049
|
+
if (demandEnabled(item.demand) || item.requestedAt !== null) {
|
|
10050
|
+
const queued = item.queuedRefresh;
|
|
10051
|
+
item.queuedRefresh = null;
|
|
10052
|
+
await startPoll(item, { manual: item.invalidations.size > 0 || item.requestedAt !== null || Boolean(queued),
|
|
10053
|
+
force: queued?.force === true });
|
|
9023
10054
|
}
|
|
9024
10055
|
} catch (error) {
|
|
9025
10056
|
try { onDiagnostic({ stage: "initialize", reason: redact(shortErr(error)) }); } catch { /* diagnostics are isolated */ }
|
|
9026
|
-
item.onHold?.("
|
|
10057
|
+
item.onHold?.(config.providers[item.target.provider]?.type === "github-app" &&
|
|
10058
|
+
error?.message === "GitHub App token unavailable" ? "app-auth" : "disconnected");
|
|
9027
10059
|
schedule(item, 1_000, () => startInitialize(item));
|
|
9028
10060
|
}
|
|
9029
10061
|
}
|
|
@@ -9040,7 +10072,7 @@ function createCollectorAcquisitionRuntime({
|
|
|
9040
10072
|
if (!normalizedDemand) throw new Error("invalid collector demand");
|
|
9041
10073
|
const item = { target, resource, demand: normalizedDemand, onSnapshot, onHold, id: null, identity: null,
|
|
9042
10074
|
provider: null, polling: false, pollPhase: null, pollPromise: null, controller: null,
|
|
9043
|
-
initializePromise: null, closed: false, timer: null, queuedRefresh: null,
|
|
10075
|
+
initializePromise: null, closed: false, timer: null, queuedRefresh: null, requestedAt: null,
|
|
9044
10076
|
bindingRevision: 0, invalidations: new Set(), lastDeliveredGeneration: null,
|
|
9045
10077
|
lastDeliveredSuccessAt: null, promotionGeneration: null, promotionAt: null };
|
|
9046
10078
|
items.add(item);
|
|
@@ -9349,7 +10381,14 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9349
10381
|
},
|
|
9350
10382
|
});
|
|
9351
10383
|
const write = (message) => {
|
|
9352
|
-
if (welcomed
|
|
10384
|
+
if (!welcomed || !socket?.writable) return false;
|
|
10385
|
+
socket.write(encodeCollectorFrame(message));
|
|
10386
|
+
return true;
|
|
10387
|
+
};
|
|
10388
|
+
const flushRefresh = (item) => {
|
|
10389
|
+
if (item.pendingRefresh !== null && write({ type: "refresh", id: item.id, force: item.pendingRefresh })) {
|
|
10390
|
+
item.pendingRefresh = null;
|
|
10391
|
+
}
|
|
9353
10392
|
};
|
|
9354
10393
|
const resubscribe = (resetGenerations) => {
|
|
9355
10394
|
for (const item of subscriptions.values()) {
|
|
@@ -9359,6 +10398,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9359
10398
|
}
|
|
9360
10399
|
write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
|
|
9361
10400
|
resource: item.resource, demand: item.demand });
|
|
10401
|
+
flushRefresh(item);
|
|
9362
10402
|
}
|
|
9363
10403
|
};
|
|
9364
10404
|
const connect = () => {
|
|
@@ -9449,7 +10489,7 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9449
10489
|
return { ok: false, reason: "invalid" };
|
|
9450
10490
|
}
|
|
9451
10491
|
const item = { ...subscription,
|
|
9452
|
-
onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1,
|
|
10492
|
+
onSnapshot, onHold, onError, onDiagnostic, generation: -1, controlGeneration: -1, pendingRefresh: null,
|
|
9453
10493
|
sourceAge: createCollectorSourceAgeTracker({ now, monotonicNow, checkpoint: sourceCheckpoint }) };
|
|
9454
10494
|
subscriptions.set(id, item);
|
|
9455
10495
|
if (welcomed) write({ type: "subscribe", id: item.id, host: item.host, repo: item.repo,
|
|
@@ -9467,7 +10507,10 @@ function createCollectorProtocolClient({ createTransport, onReady = () => {},
|
|
|
9467
10507
|
write({ type: "demand", id, demand: normalized });
|
|
9468
10508
|
return { ok: true, changed: true };
|
|
9469
10509
|
},
|
|
9470
|
-
refresh(force = false) {
|
|
10510
|
+
refresh(force = false) {
|
|
10511
|
+
item.pendingRefresh = force === true || item.pendingRefresh === true;
|
|
10512
|
+
flushRefresh(item);
|
|
10513
|
+
},
|
|
9471
10514
|
inspect() { write({ type: "inspect", id }); },
|
|
9472
10515
|
close() { write({ type: "unsubscribe", id }); subscriptions.delete(id); assembler.cancel(id); },
|
|
9473
10516
|
} };
|
|
@@ -9508,6 +10551,7 @@ function createSshCollectorClient({ alias, createTransport = null, random = Math
|
|
|
9508
10551
|
}
|
|
9509
10552
|
|
|
9510
10553
|
function collectorDisplayDecision({ connected, hasSnapshot, hold = null }) {
|
|
10554
|
+
if (hold === "app-auth") return { disconnected: true, reason: "app-auth" };
|
|
9511
10555
|
if (!connected || hold === "disconnected") return { disconnected: true };
|
|
9512
10556
|
if (hold === "shared-wait") return { mode: "waiting", waitCause: "shared-lane" };
|
|
9513
10557
|
if (["observer", "primary", "secondary", "coordination"].includes(hold)) {
|
|
@@ -9947,8 +10991,10 @@ const ACQUISITION_METRIC_KEYS = [
|
|
|
9947
10991
|
const ACQUISITION_HOLD_REASONS = new Set([
|
|
9948
10992
|
"observer", "primary", "secondary", "disconnected", "coordination",
|
|
9949
10993
|
]);
|
|
9950
|
-
const ACQUISITION_STORE_VERSION =
|
|
10994
|
+
const ACQUISITION_STORE_VERSION = 2;
|
|
9951
10995
|
const ACQUISITION_MAX_UNCERTAIN_RECEIPTS = 1024;
|
|
10996
|
+
const ACQUISITION_MAX_QUOTA_PROJECTIONS = 128;
|
|
10997
|
+
const ACQUISITION_QUOTA_PROJECTION_TTL_MS = 60_000;
|
|
9952
10998
|
|
|
9953
10999
|
function normalizeAcquisitionDiagnosticMetadata(raw, path) {
|
|
9954
11000
|
const normalized = normalizeAcquisitionStore(raw);
|
|
@@ -10098,16 +11144,23 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10098
11144
|
if (!admitted?.ok || admitted.value.status !== "started") {
|
|
10099
11145
|
return { index, skipped: { ...skippedDoctorProbe(name, args, admitted), ...(document ? { document } : {}) } };
|
|
10100
11146
|
}
|
|
10101
|
-
return { index, name, args, operation, input, document,
|
|
11147
|
+
return { index, name, args, operation, input, document,
|
|
11148
|
+
reservationId: admitted.value.reservationId,
|
|
11149
|
+
receiptCapability: admitted.value.receiptCapability };
|
|
10102
11150
|
});
|
|
10103
11151
|
results = admittedProbes.map((item) => item.skipped ?? null);
|
|
10104
11152
|
const runnable = admittedProbes.filter((item) => !item.skipped);
|
|
10105
11153
|
const settled = await mapAllSettledBounded(runnable, 4, async (item) => {
|
|
10106
|
-
const result = await requestIdentityStorage.run(scope,
|
|
11154
|
+
const result = await requestIdentityStorage.run({ ...scope,
|
|
11155
|
+
receiptCapability: item.receiptCapability }, () => probe(item.name, item.args, item.operation, item.input));
|
|
10107
11156
|
if (item.document) result.document = item.document;
|
|
10108
|
-
|
|
11157
|
+
const terminal = queueTerminalization(scope, item.receiptCapability, result.failed
|
|
10109
11158
|
? { outcome: "rejected" }
|
|
10110
|
-
: { outcome: "measured-success",
|
|
11159
|
+
: { outcome: "measured-success", actualCosts: operationCost(item.operation) });
|
|
11160
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
11161
|
+
result.failed = true;
|
|
11162
|
+
result.stderr = `receipt settlement ${terminal.status}`;
|
|
11163
|
+
}
|
|
10111
11164
|
return result;
|
|
10112
11165
|
});
|
|
10113
11166
|
settled.forEach((outcome, index) => {
|
|
@@ -10128,9 +11181,32 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10128
11181
|
governorResult = inspectGovernor(cleanup, Date.now());
|
|
10129
11182
|
}
|
|
10130
11183
|
}
|
|
11184
|
+
const cachedQuota = !probeEndpoints ? doctorCachedQuotaScope(effectiveHost) : null;
|
|
10131
11185
|
const governor = probeEndpoints
|
|
10132
11186
|
? governorHealth(governorResult, Date.now())
|
|
10133
|
-
:
|
|
11187
|
+
: cachedQuota?.ok ? governorHealth(cachedQuota.result, Date.now())
|
|
11188
|
+
: { status: `unavailable (${cachedQuota?.reason ?? "cached scope unavailable"})`, leases: 0, resources: {} };
|
|
11189
|
+
const diagnosticScope = probeEndpoints
|
|
11190
|
+
? scopeResult?.ok ? scopeResult.value : null
|
|
11191
|
+
: cachedQuota?.ok ? { path: cachedQuota.path } : null;
|
|
11192
|
+
const recoveryJournal = diagnosticScope ? readRecoveryTransitions(diagnosticScope) : null;
|
|
11193
|
+
const recoveryDetails = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
11194
|
+
diagnosticScope ? recoveryLedgerSummary(diagnosticScope, resource) : null]));
|
|
11195
|
+
const currentRecovery = RATE_RESOURCES.map((resource) => recoveryJournal?.current[resource])
|
|
11196
|
+
.filter(Boolean).sort((left, right) => right.at - left.at)[0] ?? null;
|
|
11197
|
+
const lastFailedRecovery = [...(recoveryJournal?.entries ?? [])].reverse()
|
|
11198
|
+
.find((entry) => entry.code !== "clear") ?? null;
|
|
11199
|
+
const retryDates = RATE_RESOURCES.flatMap((resource) => {
|
|
11200
|
+
const active = recoveryJournal?.current[resource];
|
|
11201
|
+
const observer = (probeEndpoints ? governorResult : cachedQuota?.result)?.value?.observers[resource];
|
|
11202
|
+
return [active?.retryAt, observer?.outcome === "failed" ? observer.nextAt : null];
|
|
11203
|
+
}).filter((value) => Number.isFinite(value) && value > Date.now());
|
|
11204
|
+
const nextRecoveryAt = retryDates.length > 0 ? Math.min(...retryDates) : null;
|
|
11205
|
+
let recoveryReason = "unavailable";
|
|
11206
|
+
if (currentRecovery) recoveryReason = `${currentRecovery.resource}: ${currentRecovery.code}`;
|
|
11207
|
+
else if (recoveryJournal?.status === "corrupt") recoveryReason = "diagnostic journal corrupt";
|
|
11208
|
+
else if (recoveryJournal?.status === "unavailable") recoveryReason = "diagnostic journal unavailable";
|
|
11209
|
+
else if (["healthy", "missing"].includes(recoveryJournal?.status)) recoveryReason = "none recorded";
|
|
10134
11210
|
const diagnosticSource = runtime.connect === "local"
|
|
10135
11211
|
? "local collector"
|
|
10136
11212
|
: runtime.connect?.startsWith("ssh:") ? "SSH collector" : "standalone";
|
|
@@ -10175,6 +11251,10 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10175
11251
|
),
|
|
10176
11252
|
"",
|
|
10177
11253
|
...section("API governor"),
|
|
11254
|
+
field("cached scope", probeEndpoints ? "verified by probe" : cachedQuota?.ok ? "read-only cached identity" : `unavailable (${cachedQuota?.reason ?? "unknown"})`),
|
|
11255
|
+
...(cachedQuota?.path && !cachedQuota.ok
|
|
11256
|
+
? [field("state file", `${join(dirname(cachedQuota.path), "quota-[redacted].json")} (quota fingerprint withheld)`)]
|
|
11257
|
+
: []),
|
|
10178
11258
|
field("status", governor.status),
|
|
10179
11259
|
field("live leases", governor.leases),
|
|
10180
11260
|
...RATE_RESOURCES.map((resource) => {
|
|
@@ -10186,6 +11266,21 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10186
11266
|
: "unavailable",
|
|
10187
11267
|
);
|
|
10188
11268
|
}),
|
|
11269
|
+
field("recovery reason", recoveryReason),
|
|
11270
|
+
field("last failed transition", lastFailedRecovery?.code ?? "unavailable"),
|
|
11271
|
+
...RATE_RESOURCES.flatMap((resource) => [
|
|
11272
|
+
field(`${resource} receipt units`, recoveryDetails[resource]?.receiptUnits ?? "unavailable"),
|
|
11273
|
+
field(`${resource} receipt count`, recoveryDetails[resource]?.receiptCount ?? "unavailable"),
|
|
11274
|
+
field(`${resource} debt units`, recoveryDetails[resource]?.debtUnits ?? "unavailable"),
|
|
11275
|
+
field(`${resource} debt count`, recoveryDetails[resource]?.debtCount ?? "unavailable"),
|
|
11276
|
+
field(`${resource} oldest unresolved`, recoveryDetails[resource]?.oldestUnresolvedAt == null
|
|
11277
|
+
? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].oldestUnresolvedAt))),
|
|
11278
|
+
field(`${resource} observer age`, recoveryDetails[resource]?.observerAt == null
|
|
11279
|
+
? "unavailable" : formatDuration(Math.max(0, Date.now() - recoveryDetails[resource].observerAt))),
|
|
11280
|
+
]),
|
|
11281
|
+
field("next retry", nextRecoveryAt == null ? "unavailable" : new Date(nextRecoveryAt).toISOString()),
|
|
11282
|
+
field("terminalization backlog", currentRecovery?.terminalizationBacklog == null
|
|
11283
|
+
? "unavailable" : `${currentRecovery.terminalizationBacklog} (last recorded)`),
|
|
10189
11284
|
"",
|
|
10190
11285
|
...section("Acquisition metrics"),
|
|
10191
11286
|
field("source", acquisitionDiagnostic?.source ?? "standalone"),
|
|
@@ -10202,7 +11297,8 @@ async function runDoctor({ probeEndpoints = false } = {}) {
|
|
|
10202
11297
|
field("failed requests", acquisitionDiagnostic?.metrics.failedRequests ?? 0),
|
|
10203
11298
|
field("REST actual 200/304", `${acquisitionDiagnostic?.metrics.rest200 ?? 0}/${acquisitionDiagnostic?.metrics.rest304 ?? 0}`),
|
|
10204
11299
|
field("proven actual units", `${acquisitionDiagnostic?.metrics.coreUnits ?? 0} core + ${acquisitionDiagnostic?.metrics.graphqlUnits ?? 0} GraphQL`),
|
|
10205
|
-
field("conservative outstanding",
|
|
11300
|
+
field("conservative outstanding", acquisitionOutstandingText(acquisitionDiagnostic)),
|
|
11301
|
+
field("legacy unverified", `${acquisitionDiagnostic?.legacyUnverified?.coreUnits ?? 0} core + ${acquisitionDiagnostic?.legacyUnverified?.graphqlUnits ?? 0} GraphQL (diagnostic only)`),
|
|
10206
11302
|
field("observer calls", acquisitionDiagnostic?.metrics.observerCalls ?? 0),
|
|
10207
11303
|
field("cache/followers", `${acquisitionDiagnostic?.metrics.cacheHits ?? 0}/${acquisitionDiagnostic?.metrics.joinedFollowers ?? 0}`),
|
|
10208
11304
|
field("queue wait (measured)", `${acquisitionDiagnostic?.metrics.queueWaitMs ?? 0}ms`),
|
|
@@ -10321,7 +11417,7 @@ function parseRepoTarget(value) {
|
|
|
10321
11417
|
// in-flight guard absorbs every other one and the effective rate is whatever
|
|
10322
11418
|
// `gh` can sustain -- the requested interval silently stops being real. Clamping
|
|
10323
11419
|
// with a stated minimum is honest where silently accepting it would not be.
|
|
10324
|
-
const BACKGROUND_MODES = ["all", "off"];
|
|
11420
|
+
const BACKGROUND_MODES = ["all", "off", "on-demand"];
|
|
10325
11421
|
const MIN_REFRESH_SECONDS = 2;
|
|
10326
11422
|
const MAX_REFRESH_SECONDS = 3600;
|
|
10327
11423
|
|
|
@@ -10551,7 +11647,11 @@ Options:
|
|
|
10551
11647
|
and running Actions are checked every ${POLL_ACTIVE_CI_MS / 1000}s.
|
|
10552
11648
|
Safe shared grants may run later.
|
|
10553
11649
|
--tab <name> Tab to start on: ${TAB_KEYS.join(", ")}.
|
|
10554
|
-
--background <mode> ${BACKGROUND_MODES.join(" or ")} (default
|
|
11650
|
+
--background <mode> ${BACKGROUND_MODES.join(" or ")} (default on-demand).
|
|
11651
|
+
on-demand watches active Actions; other tabs refresh
|
|
11652
|
+
when opened or with r, then keep cached results.
|
|
11653
|
+
all polls every tab; off polls the selected tab.
|
|
11654
|
+
\`off\` never requests data
|
|
10555
11655
|
for a tab you are not looking at; its count stays at
|
|
10556
11656
|
the last known value and visibly ages.
|
|
10557
11657
|
--verbose Write one line per gh invocation to stderr. stderr
|
|
@@ -10579,9 +11679,11 @@ Run it from inside a locally cloned GitHub repository; the repo is inferred
|
|
|
10579
11679
|
from the git remote, the same way \`gh\` does it. Requires the \`gh\` CLI
|
|
10580
11680
|
(2.20 or newer), authenticated via \`gh auth login\`.
|
|
10581
11681
|
|
|
10582
|
-
|
|
11682
|
+
By default, active Actions is considered every ${REFRESH_MS / 1000}s while its
|
|
10583
11683
|
content is changing, then every ${POLL_QUIET_MS.actions / 1000}s once two checks in a row come back
|
|
10584
|
-
unchanged
|
|
11684
|
+
unchanged. Secondary tabs check on opening or r and retain cached results.
|
|
11685
|
+
With --background all, inactive tabs are considered about every
|
|
11686
|
+
${Math.max(REFRESH_MS * BACKGROUND_EVERY, POLL_BACKGROUND_MS.actions) / 1000}s (Security is slower). Quota-consuming
|
|
10585
11687
|
calls still need a shared, resource-specific grant that preserves a hard reserve.
|
|
10586
11688
|
Scheduled checks and manual refresh do not bypass that safety check: neither
|
|
10587
11689
|
refresh key can spend past the reserve.
|
|
@@ -12326,6 +13428,8 @@ const ACQUISITION_MAX_ENTITY_BYTES = 1024 * 1024;
|
|
|
12326
13428
|
const ACQUISITION_MAX_ENTITIES = 512;
|
|
12327
13429
|
const ACQUISITION_MAX_LIVE_TARGETS = 32;
|
|
12328
13430
|
const ACQUISITION_MAX_SUBSCRIPTIONS = 128;
|
|
13431
|
+
const ACQUISITION_READ_MAX_ATTEMPTS = 24;
|
|
13432
|
+
const ACQUISITION_READ_MAX_MS = 1_500;
|
|
12329
13433
|
let acquisitionTempSequence = 0;
|
|
12330
13434
|
|
|
12331
13435
|
function acquisitionStorePath(options = {}) {
|
|
@@ -12337,7 +13441,8 @@ function emptyAcquisitionStore() {
|
|
|
12337
13441
|
version: ACQUISITION_STORE_VERSION,
|
|
12338
13442
|
producerEpoch: governorId(),
|
|
12339
13443
|
metrics: emptyAcquisitionMetrics(),
|
|
12340
|
-
|
|
13444
|
+
quotaProjections: {},
|
|
13445
|
+
legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
|
|
12341
13446
|
subscriptions: {},
|
|
12342
13447
|
queries: {},
|
|
12343
13448
|
aliases: {},
|
|
@@ -12365,32 +13470,6 @@ function acquisitionMetricState(state) {
|
|
|
12365
13470
|
return state.metrics;
|
|
12366
13471
|
}
|
|
12367
13472
|
|
|
12368
|
-
function removeAcquisitionReceipt(state, id) {
|
|
12369
|
-
const receipt = state.uncertainReceipts?.[id];
|
|
12370
|
-
if (!receipt) return false;
|
|
12371
|
-
const metrics = acquisitionMetricState(state);
|
|
12372
|
-
const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
|
|
12373
|
-
metrics[key] = Math.max(0, metrics[key] - receipt.units);
|
|
12374
|
-
delete state.uncertainReceipts[id];
|
|
12375
|
-
for (const record of Object.values(state.queries ?? {})) {
|
|
12376
|
-
if (record.claim?.receiptIds?.includes(id)) {
|
|
12377
|
-
record.claim.receiptIds = record.claim.receiptIds.filter((receiptId) => receiptId !== id);
|
|
12378
|
-
}
|
|
12379
|
-
}
|
|
12380
|
-
return true;
|
|
12381
|
-
}
|
|
12382
|
-
|
|
12383
|
-
function addAcquisitionReceipt(state, receipt) {
|
|
12384
|
-
state.uncertainReceipts ??= {};
|
|
12385
|
-
if (state.uncertainReceipts[receipt.id]) return true;
|
|
12386
|
-
if (Object.keys(state.uncertainReceipts).length >= ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return false;
|
|
12387
|
-
state.uncertainReceipts[receipt.id] = receipt;
|
|
12388
|
-
const metrics = acquisitionMetricState(state);
|
|
12389
|
-
const key = receipt.resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
|
|
12390
|
-
metrics[key] += receipt.units;
|
|
12391
|
-
return true;
|
|
12392
|
-
}
|
|
12393
|
-
|
|
12394
13473
|
function normalizeAcquisitionReceipt(raw, expectedId = raw?.id) {
|
|
12395
13474
|
if (!isRecord(raw) || raw.id !== expectedId || typeof raw.id !== "string" || raw.id.length > 240 ||
|
|
12396
13475
|
typeof raw.reservationId !== "string" || raw.reservationId.length < 1 ||
|
|
@@ -12415,63 +13494,6 @@ function addAcquisitionRequestMetrics(state, delta) {
|
|
|
12415
13494
|
return addAcquisitionMetrics(state, observed);
|
|
12416
13495
|
}
|
|
12417
13496
|
|
|
12418
|
-
function releaseAcquisitionUncertainty(state, record) {
|
|
12419
|
-
if (!record?.claim?.started) return;
|
|
12420
|
-
const receiptIds = record.claim.receiptIds ?? [];
|
|
12421
|
-
if (Object.hasOwn(record.claim, "receiptIds")) {
|
|
12422
|
-
for (const id of receiptIds) removeAcquisitionReceipt(state, id);
|
|
12423
|
-
return;
|
|
12424
|
-
}
|
|
12425
|
-
const costs = tabRequestCost(record.query.resource);
|
|
12426
|
-
const metrics = acquisitionMetricState(state);
|
|
12427
|
-
metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - costs.core);
|
|
12428
|
-
metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - costs.graphql);
|
|
12429
|
-
}
|
|
12430
|
-
|
|
12431
|
-
function releaseKnownAcquisitionUncertainty(state, record, requestMetrics) {
|
|
12432
|
-
if (!record?.claim?.started) return;
|
|
12433
|
-
const receiptIds = record.claim.receiptIds ?? [];
|
|
12434
|
-
if (Object.hasOwn(record.claim, "receiptIds")) {
|
|
12435
|
-
for (const resource of RATE_RESOURCES) {
|
|
12436
|
-
const metricKey = resource === "core" ? "coreUnits" : "graphqlUnits";
|
|
12437
|
-
const uncertainKey = resource === "core" ? "uncertainCoreUnits" : "uncertainGraphqlUnits";
|
|
12438
|
-
const resourceReceipts = receiptIds
|
|
12439
|
-
.map((id) => state.uncertainReceipts?.[id])
|
|
12440
|
-
.filter((receipt) => receipt?.resource === resource);
|
|
12441
|
-
if (Object.hasOwn(requestMetrics, uncertainKey)) {
|
|
12442
|
-
let remaining = requestMetrics[uncertainKey];
|
|
12443
|
-
for (const receipt of resourceReceipts) {
|
|
12444
|
-
const retained = Math.min(receipt.units, remaining);
|
|
12445
|
-
const released = receipt.units - retained;
|
|
12446
|
-
remaining -= retained;
|
|
12447
|
-
if (retained === 0) removeAcquisitionReceipt(state, receipt.id);
|
|
12448
|
-
else if (released > 0) {
|
|
12449
|
-
receipt.units = retained;
|
|
12450
|
-
acquisitionMetricState(state)[uncertainKey] -= released;
|
|
12451
|
-
}
|
|
12452
|
-
}
|
|
12453
|
-
continue;
|
|
12454
|
-
}
|
|
12455
|
-
if (Object.hasOwn(requestMetrics, metricKey)) {
|
|
12456
|
-
for (const receipt of resourceReceipts) removeAcquisitionReceipt(state, receipt.id);
|
|
12457
|
-
}
|
|
12458
|
-
}
|
|
12459
|
-
return;
|
|
12460
|
-
}
|
|
12461
|
-
const declared = tabRequestCost(record.query.resource);
|
|
12462
|
-
const metrics = acquisitionMetricState(state);
|
|
12463
|
-
if (Object.hasOwn(requestMetrics, "uncertainCoreUnits")) {
|
|
12464
|
-
metrics.uncertainCoreUnits = Math.min(declared.core, requestMetrics.uncertainCoreUnits);
|
|
12465
|
-
} else if (Object.hasOwn(requestMetrics, "coreUnits")) {
|
|
12466
|
-
metrics.uncertainCoreUnits = Math.max(0, metrics.uncertainCoreUnits - declared.core);
|
|
12467
|
-
}
|
|
12468
|
-
if (Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
|
|
12469
|
-
metrics.uncertainGraphqlUnits = Math.min(declared.graphql, requestMetrics.uncertainGraphqlUnits);
|
|
12470
|
-
} else if (Object.hasOwn(requestMetrics, "graphqlUnits")) {
|
|
12471
|
-
metrics.uncertainGraphqlUnits = Math.max(0, metrics.uncertainGraphqlUnits - declared.graphql);
|
|
12472
|
-
}
|
|
12473
|
-
}
|
|
12474
|
-
|
|
12475
13497
|
function settleAcquisitionWaiters(state, queryKey, generation, at) {
|
|
12476
13498
|
let queueWaitMs = 0;
|
|
12477
13499
|
for (const subscription of Object.values(state.subscriptions)) {
|
|
@@ -12490,8 +13512,6 @@ function settleInvalidAcquisitionPublication(state, record, claim, requestMetric
|
|
|
12490
13512
|
...normalized,
|
|
12491
13513
|
failedRequests: Math.max(1, normalized.failedRequests ?? 0),
|
|
12492
13514
|
});
|
|
12493
|
-
releaseKnownAcquisitionUncertainty(state, record, normalized);
|
|
12494
|
-
for (const receipt of receipts ?? []) addAcquisitionReceipt(state, receipt);
|
|
12495
13515
|
settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, at);
|
|
12496
13516
|
record.claim = null;
|
|
12497
13517
|
record.hold = { reason: "primary", at };
|
|
@@ -12528,9 +13548,18 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
|
|
|
12528
13548
|
subscriptionsByQuery.set(subscription.queryKey, grouped);
|
|
12529
13549
|
}
|
|
12530
13550
|
const activeQueryKeys = new Set(subscriptions.map((subscription) => subscription.queryKey));
|
|
13551
|
+
const outstandingQuota = Object.entries(state?.quotaProjections ?? {}).map(([scopeHash, projection]) => {
|
|
13552
|
+
const current = projection.status === "current" && projection.at <= nowMs &&
|
|
13553
|
+
nowMs - projection.at <= ACQUISITION_QUOTA_PROJECTION_TTL_MS;
|
|
13554
|
+
return { scopeHash, revision: projection.revision,
|
|
13555
|
+
status: current ? "current" : "unavailable", units: current ? projection.units : null,
|
|
13556
|
+
at: projection.at, accessKeys: projection.accessKeys };
|
|
13557
|
+
});
|
|
12531
13558
|
const queries = Object.values(state?.queries ?? {}).map((record) => {
|
|
12532
13559
|
const snapshot = record.snapshot;
|
|
12533
13560
|
const consumers = subscriptionsByQuery.get(record.query.queryKey) ?? [];
|
|
13561
|
+
const projection = outstandingQuota.find((item) =>
|
|
13562
|
+
item.accessKeys.includes(record.query.accessKey) && item.status === "current");
|
|
12534
13563
|
return {
|
|
12535
13564
|
resource: record.query.resource,
|
|
12536
13565
|
lastSuccessAt: snapshot?.lastSuccessAt ?? null,
|
|
@@ -12543,8 +13572,9 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
|
|
|
12543
13572
|
graphql: state?.metrics?.graphqlUnits ?? 0,
|
|
12544
13573
|
},
|
|
12545
13574
|
uncertainCost: {
|
|
12546
|
-
|
|
12547
|
-
|
|
13575
|
+
status: projection ? "current" : "unavailable",
|
|
13576
|
+
core: projection?.units.core ?? null,
|
|
13577
|
+
graphql: projection?.units.graphql ?? null,
|
|
12548
13578
|
},
|
|
12549
13579
|
coalescedConsumers: consumers.length,
|
|
12550
13580
|
};
|
|
@@ -12556,12 +13586,31 @@ function acquisitionDiagnostics(state, { source = "standalone", nowMs = Date.now
|
|
|
12556
13586
|
activeQueries: activeQueryKeys.size,
|
|
12557
13587
|
activeSubscribers: subscriptions.length,
|
|
12558
13588
|
metrics: normalizeAcquisitionMetrics(state?.metrics) ?? emptyAcquisitionMetrics(),
|
|
13589
|
+
outstandingQuota: outstandingQuota.map(({ accessKeys: _accessKeys, ...projection }) => projection),
|
|
13590
|
+
legacyUnverified: state?.legacyUnverified ?? { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
|
|
12559
13591
|
queries,
|
|
12560
13592
|
};
|
|
12561
13593
|
}
|
|
12562
13594
|
|
|
13595
|
+
function acquisitionOutstandingText(diagnostic) {
|
|
13596
|
+
const scopes = diagnostic?.outstandingQuota ?? [];
|
|
13597
|
+
if (scopes.length === 0 || scopes.some((scope) => scope.status !== "current")) {
|
|
13598
|
+
return "unavailable (quota projection pending)";
|
|
13599
|
+
}
|
|
13600
|
+
const units = scopes.reduce((total, scope) => ({
|
|
13601
|
+
core: total.core + scope.units.core,
|
|
13602
|
+
graphql: total.graphql + scope.units.graphql,
|
|
13603
|
+
}), { core: 0, graphql: 0 });
|
|
13604
|
+
return Number.isSafeInteger(units.core) && Number.isSafeInteger(units.graphql)
|
|
13605
|
+
? `${units.core} core + ${units.graphql} GraphQL across ${scopes.length} quota scopes`
|
|
13606
|
+
: "unavailable (accounting overflow)";
|
|
13607
|
+
}
|
|
13608
|
+
|
|
12563
13609
|
function acquisitionFailureHold(error) {
|
|
12564
13610
|
if (typeof error?.providerCapability === "string") return "observer";
|
|
13611
|
+
if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) {
|
|
13612
|
+
return HTTP_PERMIT_FAILURE_HOLDS[error.coordinationReason];
|
|
13613
|
+
}
|
|
12565
13614
|
if (SECONDARY_LIMIT_PATTERN.test(errText(error))) return "secondary";
|
|
12566
13615
|
if (isRateLimited(error) || /API budget paused/i.test(errText(error))) return "primary";
|
|
12567
13616
|
const attemptedHttp = Number.isSafeInteger(error?.requestMetrics?.httpRequests) &&
|
|
@@ -12889,32 +13938,58 @@ function prepareAcquisitionSnapshot(queryKey, snapshot) {
|
|
|
12889
13938
|
}
|
|
12890
13939
|
|
|
12891
13940
|
function normalizeAcquisitionStore(raw) {
|
|
12892
|
-
if (!isRecord(raw) || raw.version
|
|
13941
|
+
if (!isRecord(raw) || ![1, ACQUISITION_STORE_VERSION].includes(raw.version) || !validGovernorId(raw.producerEpoch) ||
|
|
12893
13942
|
!isRecord(raw.subscriptions) || !isRecord(raw.queries) || !isRecord(raw.aliases)) return null;
|
|
13943
|
+
const legacy = raw.version === 1;
|
|
12894
13944
|
const state = {
|
|
12895
13945
|
version: ACQUISITION_STORE_VERSION,
|
|
12896
|
-
producerEpoch: raw.producerEpoch,
|
|
13946
|
+
producerEpoch: legacy ? governorId() : raw.producerEpoch,
|
|
12897
13947
|
metrics: normalizeAcquisitionMetrics(raw.metrics),
|
|
12898
|
-
|
|
13948
|
+
quotaProjections: {},
|
|
13949
|
+
legacyUnverified: { coreUnits: 0, graphqlUnits: 0, receiptCount: 0 },
|
|
12899
13950
|
subscriptions: {},
|
|
12900
13951
|
queries: {},
|
|
12901
13952
|
aliases: {},
|
|
12902
13953
|
};
|
|
12903
13954
|
if (state.metrics === null) return null;
|
|
12904
|
-
|
|
12905
|
-
|
|
12906
|
-
|
|
12907
|
-
const
|
|
12908
|
-
|
|
12909
|
-
|
|
12910
|
-
|
|
12911
|
-
|
|
12912
|
-
|
|
12913
|
-
|
|
12914
|
-
|
|
12915
|
-
}
|
|
12916
|
-
if (
|
|
12917
|
-
|
|
13955
|
+
if (legacy) {
|
|
13956
|
+
const receipts = raw.uncertainReceipts ?? {};
|
|
13957
|
+
if (!isRecord(receipts) || Object.keys(receipts).length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return null;
|
|
13958
|
+
for (const [id, receipt] of Object.entries(receipts)) {
|
|
13959
|
+
const normalized = normalizeAcquisitionReceipt(receipt, id);
|
|
13960
|
+
if (!normalized) return null;
|
|
13961
|
+
const key = receipt.resource === "core" ? "coreUnits" : "graphqlUnits";
|
|
13962
|
+
const next = state.legacyUnverified[key] + receipt.units;
|
|
13963
|
+
if (!Number.isSafeInteger(next)) return null;
|
|
13964
|
+
state.legacyUnverified[key] = next;
|
|
13965
|
+
state.legacyUnverified.receiptCount += 1;
|
|
13966
|
+
}
|
|
13967
|
+
if (raw.uncertainReceipts === undefined) {
|
|
13968
|
+
state.legacyUnverified.coreUnits = state.metrics.uncertainCoreUnits;
|
|
13969
|
+
state.legacyUnverified.graphqlUnits = state.metrics.uncertainGraphqlUnits;
|
|
13970
|
+
} else if (state.metrics.uncertainCoreUnits !== state.legacyUnverified.coreUnits ||
|
|
13971
|
+
state.metrics.uncertainGraphqlUnits !== state.legacyUnverified.graphqlUnits) return null;
|
|
13972
|
+
state.metrics.uncertainCoreUnits = 0;
|
|
13973
|
+
state.metrics.uncertainGraphqlUnits = 0;
|
|
13974
|
+
} else {
|
|
13975
|
+
if (!isRecord(raw.quotaProjections) || !isRecord(raw.legacyUnverified) ||
|
|
13976
|
+
Object.keys(raw.quotaProjections).length > ACQUISITION_MAX_QUOTA_PROJECTIONS ||
|
|
13977
|
+
!exactKeys(raw.legacyUnverified, ["coreUnits", "graphqlUnits", "receiptCount"]) ||
|
|
13978
|
+
Object.values(raw.legacyUnverified).some((value) => !Number.isSafeInteger(value) || value < 0)) return null;
|
|
13979
|
+
state.legacyUnverified = { ...raw.legacyUnverified };
|
|
13980
|
+
for (const [scopeHash, projection] of Object.entries(raw.quotaProjections)) {
|
|
13981
|
+
if (!/^[0-9a-f]{64}$/.test(scopeHash) || !exactKeys(projection, ["revision", "units", "at", "status", "accessKeys"]) ||
|
|
13982
|
+
!Number.isSafeInteger(projection.revision) || projection.revision < 0 ||
|
|
13983
|
+
!exactKeys(projection.units, RATE_RESOURCES) ||
|
|
13984
|
+
Object.values(projection.units).some((value) => !Number.isSafeInteger(value) || value < 0) ||
|
|
13985
|
+
!Number.isFinite(projection.at) || projection.at < 0 ||
|
|
13986
|
+
!["current", "unavailable"].includes(projection.status) ||
|
|
13987
|
+
!Array.isArray(projection.accessKeys) || projection.accessKeys.length > ACQUISITION_MAX_SUBSCRIPTIONS ||
|
|
13988
|
+
new Set(projection.accessKeys).size !== projection.accessKeys.length ||
|
|
13989
|
+
projection.accessKeys.some((key) => !/^[0-9a-f]{64}$/.test(key))) return null;
|
|
13990
|
+
state.quotaProjections[scopeHash] = { ...projection, units: { ...projection.units },
|
|
13991
|
+
accessKeys: [...projection.accessKeys] };
|
|
13992
|
+
}
|
|
12918
13993
|
}
|
|
12919
13994
|
for (const [key, value] of Object.entries(raw.aliases)) {
|
|
12920
13995
|
if (typeof key !== "string" || typeof value !== "string") return null;
|
|
@@ -12933,11 +14008,15 @@ function normalizeAcquisitionStore(raw) {
|
|
|
12933
14008
|
candidate.generation !== value.generation + 1 || !Number.isFinite(candidate.claimedAt) ||
|
|
12934
14009
|
!Number.isFinite(candidate.leaseUntil) || candidate.leaseUntil < candidate.claimedAt ||
|
|
12935
14010
|
typeof candidate.started !== "boolean") return null;
|
|
12936
|
-
|
|
12937
|
-
|
|
12938
|
-
|
|
12939
|
-
|
|
12940
|
-
|
|
14011
|
+
if (legacy) {
|
|
14012
|
+
const receipts = raw.uncertainReceipts ?? {};
|
|
14013
|
+
const receiptIds = candidate.receiptIds ?? [];
|
|
14014
|
+
if (!Array.isArray(receiptIds) || receiptIds.some((id) => typeof id !== "string" || !receipts[id])) return null;
|
|
14015
|
+
claim = null; // v1 producer authority is fenced by the new epoch.
|
|
14016
|
+
} else {
|
|
14017
|
+
if (Object.hasOwn(candidate, "receiptIds")) return null;
|
|
14018
|
+
claim = { ...candidate };
|
|
14019
|
+
}
|
|
12941
14020
|
}
|
|
12942
14021
|
const snapshot = value.snapshot === null ? null
|
|
12943
14022
|
: value.snapshot?.artifact
|
|
@@ -12990,37 +14069,63 @@ function loadAcquisitionMetadata(path) {
|
|
|
12990
14069
|
}
|
|
12991
14070
|
}
|
|
12992
14071
|
|
|
12993
|
-
function readAcquisitionSnapshot(path, record) {
|
|
12994
|
-
|
|
12995
|
-
|
|
12996
|
-
|
|
12997
|
-
|
|
12998
|
-
|
|
12999
|
-
|
|
13000
|
-
return { ok:
|
|
14072
|
+
function readAcquisitionSnapshot(path, record, { deadline = performance.now() + ACQUISITION_READ_MAX_MS } = {}) {
|
|
14073
|
+
let current = record;
|
|
14074
|
+
for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
|
|
14075
|
+
performance.now() < deadline; attempt += 1) {
|
|
14076
|
+
if (!current?.snapshot) return current === record ? { ok: true, value: null }
|
|
14077
|
+
: { ok: false, reason: "stale" };
|
|
14078
|
+
if (Array.isArray(current.snapshot.rows)) {
|
|
14079
|
+
return { ok: true, value: current.snapshot, replaced: current !== record };
|
|
14080
|
+
}
|
|
14081
|
+
try {
|
|
14082
|
+
const raw = readFileSync(join(acquisitionSnapshotRoot(path), current.snapshot.artifact), "utf8");
|
|
14083
|
+
if (Buffer.byteLength(raw) !== current.snapshot.bytes ||
|
|
14084
|
+
createHash("sha256").update(raw).digest("hex") !== current.snapshot.digest) {
|
|
14085
|
+
return { ok: false, reason: "corrupt" };
|
|
14086
|
+
}
|
|
14087
|
+
const parsed = JSON.parse(raw);
|
|
14088
|
+
const snapshot = normalizeAcquisitionSnapshot(
|
|
14089
|
+
parsed, current.query, current.generation, parsed.producerEpoch,
|
|
14090
|
+
);
|
|
14091
|
+
return snapshot ? { ok: true, value: snapshot, replaced: current !== record }
|
|
14092
|
+
: { ok: false, reason: "corrupt" };
|
|
14093
|
+
} catch (error) {
|
|
14094
|
+
if (error?.code !== "ENOENT") return { ok: false, reason: "corrupt", error };
|
|
14095
|
+
const latest = loadAcquisitionMetadata(path);
|
|
14096
|
+
if (!latest.ok) return latest;
|
|
14097
|
+
const replacement = latest.value.queries[current.query.queryKey];
|
|
14098
|
+
if (!replacement) return { ok: false, reason: "stale" };
|
|
14099
|
+
if (replacement.snapshot?.artifact === current.snapshot.artifact) {
|
|
14100
|
+
return { ok: false, reason: "corrupt", error };
|
|
14101
|
+
}
|
|
14102
|
+
current = replacement;
|
|
13001
14103
|
}
|
|
13002
|
-
const parsed = JSON.parse(raw);
|
|
13003
|
-
const snapshot = normalizeAcquisitionSnapshot(
|
|
13004
|
-
parsed, record.query, record.generation, parsed.producerEpoch,
|
|
13005
|
-
);
|
|
13006
|
-
return snapshot ? { ok: true, value: snapshot } : { ok: false, reason: "corrupt" };
|
|
13007
|
-
} catch (error) {
|
|
13008
|
-
return { ok: false, reason: "corrupt", error };
|
|
13009
14104
|
}
|
|
14105
|
+
return { ok: false, reason: "busy" };
|
|
13010
14106
|
}
|
|
13011
14107
|
|
|
13012
|
-
function hydrateAcquisitionStore(path, state) {
|
|
14108
|
+
function hydrateAcquisitionStore(path, state, deadline) {
|
|
13013
14109
|
for (const record of Object.values(state.queries)) {
|
|
13014
|
-
const loaded = readAcquisitionSnapshot(path, record);
|
|
14110
|
+
const loaded = readAcquisitionSnapshot(path, record, { deadline });
|
|
13015
14111
|
if (!loaded.ok) return loaded;
|
|
14112
|
+
if (loaded.replaced) return { ok: false, reason: "stale" };
|
|
13016
14113
|
record.snapshot = loaded.value;
|
|
13017
14114
|
}
|
|
13018
14115
|
return { ok: true, value: state };
|
|
13019
14116
|
}
|
|
13020
14117
|
|
|
13021
|
-
function loadAcquisitionStore(path) {
|
|
13022
|
-
const
|
|
13023
|
-
|
|
14118
|
+
function loadAcquisitionStore(path, { onMetadataLoaded = null } = {}) {
|
|
14119
|
+
const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
|
|
14120
|
+
for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
|
|
14121
|
+
performance.now() < deadline; attempt += 1) {
|
|
14122
|
+
const loaded = loadAcquisitionMetadata(path);
|
|
14123
|
+
if (!loaded.ok) return loaded;
|
|
14124
|
+
onMetadataLoaded?.();
|
|
14125
|
+
const hydrated = hydrateAcquisitionStore(path, loaded.value, deadline);
|
|
14126
|
+
if (hydrated.reason !== "stale") return hydrated;
|
|
14127
|
+
}
|
|
14128
|
+
return { ok: false, reason: "busy" };
|
|
13024
14129
|
}
|
|
13025
14130
|
|
|
13026
14131
|
function writeAcquisitionArtifact(root, artifact, payload) {
|
|
@@ -13193,8 +14298,8 @@ function trimAcquisitionStore(state, protectedQueryKey = null) {
|
|
|
13193
14298
|
|
|
13194
14299
|
async function runStartedAcquisitionTransport(markStarted, transport) {
|
|
13195
14300
|
const started = await markStarted();
|
|
13196
|
-
if (!started.ok) return started;
|
|
13197
|
-
if (started.value?.status !== "started") return { ok: false, reason: "already-started" };
|
|
14301
|
+
if (!started.ok) return { ...started, notStarted: true };
|
|
14302
|
+
if (started.value?.status !== "started") return { ok: false, reason: "already-started", notStarted: true };
|
|
13198
14303
|
return { ok: true, value: await transport() };
|
|
13199
14304
|
}
|
|
13200
14305
|
|
|
@@ -13244,7 +14349,7 @@ function createAcquisitionEngine({
|
|
|
13244
14349
|
}
|
|
13245
14350
|
const loaded = loadRecordSnapshot(record);
|
|
13246
14351
|
if (!loaded.ok) return loaded;
|
|
13247
|
-
item.generation =
|
|
14352
|
+
item.generation = loaded.value.generation;
|
|
13248
14353
|
try { item.callback?.(loaded.value); } catch { /* subscriber callback is isolated */ }
|
|
13249
14354
|
return { ok: true, value: true, snapshot: loaded.value };
|
|
13250
14355
|
}
|
|
@@ -13277,7 +14382,8 @@ function createAcquisitionEngine({
|
|
|
13277
14382
|
const record = loaded.value.queries[item.queryKey] ?? null;
|
|
13278
14383
|
if (!record?.snapshot) return { ok: true, value: record };
|
|
13279
14384
|
const snapshot = loadRecordSnapshot(record);
|
|
13280
|
-
return snapshot.ok ? { ok: true, value: { ...record,
|
|
14385
|
+
return snapshot.ok ? { ok: true, value: { ...record,
|
|
14386
|
+
generation: snapshot.value?.generation ?? record.generation, snapshot: snapshot.value } } : snapshot;
|
|
13281
14387
|
}
|
|
13282
14388
|
|
|
13283
14389
|
function heartbeat() {
|
|
@@ -13445,14 +14551,38 @@ function createAcquisitionEngine({
|
|
|
13445
14551
|
aliasConflict: matching.length > 1 } };
|
|
13446
14552
|
});
|
|
13447
14553
|
if (!result.ok) return result;
|
|
13448
|
-
|
|
13449
|
-
|
|
13450
|
-
|
|
13451
|
-
|
|
13452
|
-
local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation:
|
|
14554
|
+
// A publisher can canonicalize the query and remap this just-committed
|
|
14555
|
+
// shared subscription before its snapshot is hydrated. Register locally
|
|
14556
|
+
// first so every failure still has a cleanup owner, then follow the
|
|
14557
|
+
// subscription ID in fresh metadata rather than the old query key.
|
|
14558
|
+
local.set(id, { nonce, queryKey: result.value.queryKey, callback, generation: 0,
|
|
13453
14559
|
claimNonce: null, pending: null });
|
|
13454
14560
|
ensureTimer();
|
|
13455
|
-
|
|
14561
|
+
const deadline = performance.now() + ACQUISITION_READ_MAX_MS;
|
|
14562
|
+
for (let attempt = 0; attempt < ACQUISITION_READ_MAX_ATTEMPTS &&
|
|
14563
|
+
performance.now() < deadline; attempt += 1) {
|
|
14564
|
+
const latest = load();
|
|
14565
|
+
if (!latest.ok) { unsubscribe(id); return latest; }
|
|
14566
|
+
const shared = latest.value.subscriptions[id];
|
|
14567
|
+
if (!shared || shared.nonce !== nonce) { unsubscribe(id); return { ok: false, reason: "stale" }; }
|
|
14568
|
+
const item = local.get(id);
|
|
14569
|
+
item.queryKey = shared.queryKey;
|
|
14570
|
+
const record = latest.value.queries[shared.queryKey];
|
|
14571
|
+
if (!record) continue;
|
|
14572
|
+
const loadedSnapshot = loadRecordSnapshot(record);
|
|
14573
|
+
if (!loadedSnapshot.ok) {
|
|
14574
|
+
if (["stale", "busy"].includes(loadedSnapshot.reason)) continue;
|
|
14575
|
+
unsubscribe(id);
|
|
14576
|
+
return loadedSnapshot;
|
|
14577
|
+
}
|
|
14578
|
+
if (loadedSnapshot.replaced) continue;
|
|
14579
|
+
result.value.queryKey = shared.queryKey;
|
|
14580
|
+
result.value.snapshot = loadedSnapshot.value;
|
|
14581
|
+
item.generation = loadedSnapshot.value?.generation ?? record.generation;
|
|
14582
|
+
return result;
|
|
14583
|
+
}
|
|
14584
|
+
unsubscribe(id);
|
|
14585
|
+
return { ok: false, reason: "busy" };
|
|
13456
14586
|
}
|
|
13457
14587
|
|
|
13458
14588
|
function updateDemand(id, demandInput) {
|
|
@@ -13509,11 +14639,6 @@ function createAcquisitionEngine({
|
|
|
13509
14639
|
const requestMetrics = produced?.requestMetrics === undefined
|
|
13510
14640
|
? {}
|
|
13511
14641
|
: normalizeAcquisitionMetrics(produced.requestMetrics, { partial: true });
|
|
13512
|
-
const receiptInputs = produced?.uncertainReceipts ?? [];
|
|
13513
|
-
const receipts = Array.isArray(receiptInputs)
|
|
13514
|
-
? receiptInputs.map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
|
|
13515
|
-
const receiptsValid = receipts && !receipts.includes(null) &&
|
|
13516
|
-
receipts.every((receipt) => receipt.accessKey === record.query.accessKey);
|
|
13517
14642
|
const loadedPrevious = loadRecordSnapshot(record);
|
|
13518
14643
|
if (!loadedPrevious.ok) return loadedPrevious;
|
|
13519
14644
|
if (loadedPrevious.value) record.snapshot = loadedPrevious.value;
|
|
@@ -13527,20 +14652,15 @@ function createAcquisitionEngine({
|
|
|
13527
14652
|
snapshot = { ...snapshot, lastChangedAt: loadedPrevious.value.lastChangedAt };
|
|
13528
14653
|
}
|
|
13529
14654
|
if (!snapshot) {
|
|
13530
|
-
settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics,
|
|
14655
|
+
settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics, [], now());
|
|
13531
14656
|
return { value: { ok: false, reason: "invalid" } };
|
|
13532
14657
|
}
|
|
13533
14658
|
const previous = record.snapshot;
|
|
13534
14659
|
const previousClaim = record.claim;
|
|
13535
|
-
if (requestMetrics === null
|
|
14660
|
+
if (requestMetrics === null) {
|
|
13536
14661
|
settleInvalidAcquisitionPublication(state, record, effectiveClaim, requestMetrics ?? {}, [], now());
|
|
13537
14662
|
return { value: { ok: false, reason: "invalid" } };
|
|
13538
14663
|
}
|
|
13539
|
-
const receiptIds = new Set(Object.keys(state.uncertainReceipts ?? {}));
|
|
13540
|
-
const novelReceipts = receipts.filter((receipt) => !receiptIds.has(receipt.id));
|
|
13541
|
-
if (receiptIds.size + novelReceipts.length > ACQUISITION_MAX_UNCERTAIN_RECEIPTS) {
|
|
13542
|
-
return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
|
|
13543
|
-
}
|
|
13544
14664
|
record.generation = effectiveClaim.generation;
|
|
13545
14665
|
record.snapshot = snapshot;
|
|
13546
14666
|
record.claim = null;
|
|
@@ -13552,14 +14672,7 @@ function createAcquisitionEngine({
|
|
|
13552
14672
|
record.claim = previousClaim;
|
|
13553
14673
|
return { value: { ok: false, reason: "capacity", retryClaim: effectiveClaim, remapped } };
|
|
13554
14674
|
}
|
|
13555
|
-
if (Object.hasOwn(requestMetrics, "uncertainCoreUnits") ||
|
|
13556
|
-
Object.hasOwn(requestMetrics, "uncertainGraphqlUnits")) {
|
|
13557
|
-
releaseKnownAcquisitionUncertainty(state, { ...record, claim: previousClaim }, requestMetrics);
|
|
13558
|
-
} else {
|
|
13559
|
-
releaseAcquisitionUncertainty(state, { ...record, claim: previousClaim });
|
|
13560
|
-
}
|
|
13561
14675
|
addAcquisitionRequestMetrics(state, requestMetrics);
|
|
13562
|
-
for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
|
|
13563
14676
|
settleAcquisitionWaiters(state, record.query.queryKey, effectiveClaim.generation, now());
|
|
13564
14677
|
return { value: { ok: true, value: { role: "producer", snapshot,
|
|
13565
14678
|
queryKey: record.query.queryKey, remapped } } };
|
|
@@ -13613,10 +14726,8 @@ function createAcquisitionEngine({
|
|
|
13613
14726
|
function settleFailure(id, claim, failure) {
|
|
13614
14727
|
const item = local.get(id);
|
|
13615
14728
|
const requestMetrics = normalizeAcquisitionMetrics(failure?.requestMetrics ?? {}, { partial: true });
|
|
13616
|
-
const receipts = Array.isArray(failure?.uncertainReceipts ?? [])
|
|
13617
|
-
? (failure?.uncertainReceipts ?? []).map((receipt) => normalizeAcquisitionReceipt(receipt)) : null;
|
|
13618
14729
|
const hold = failure?.hold ?? "primary";
|
|
13619
|
-
if (!item || !isRecord(claim) || requestMetrics === null ||
|
|
14730
|
+
if (!item || !isRecord(claim) || requestMetrics === null ||
|
|
13620
14731
|
!ACQUISITION_HOLD_REASONS.has(hold)) {
|
|
13621
14732
|
return { ok: false, reason: "invalid" };
|
|
13622
14733
|
}
|
|
@@ -13625,13 +14736,7 @@ function createAcquisitionEngine({
|
|
|
13625
14736
|
if (!record || record.claim?.nonce !== claim.nonce || record.claim.generation !== claim.generation) {
|
|
13626
14737
|
return { value: { ok: false, reason: "stale", definitive: true } };
|
|
13627
14738
|
}
|
|
13628
|
-
if (receipts.some((receipt) => receipt.accessKey !== record.query.accessKey) ||
|
|
13629
|
-
Object.keys(state.uncertainReceipts ?? {}).length +
|
|
13630
|
-
receipts.filter((receipt) => !state.uncertainReceipts?.[receipt.id]).length >
|
|
13631
|
-
ACQUISITION_MAX_UNCERTAIN_RECEIPTS) return { ok: false, reason: "capacity" };
|
|
13632
14739
|
addAcquisitionRequestMetrics(state, requestMetrics);
|
|
13633
|
-
releaseKnownAcquisitionUncertainty(state, record, requestMetrics);
|
|
13634
|
-
for (const receipt of receipts) addAcquisitionReceipt(state, receipt);
|
|
13635
14740
|
settleAcquisitionWaiters(state, record.query.queryKey, claim.generation, now());
|
|
13636
14741
|
record.claim = null;
|
|
13637
14742
|
record.hold = { reason: hold, at: now() };
|
|
@@ -13654,7 +14759,11 @@ function createAcquisitionEngine({
|
|
|
13654
14759
|
const item = local.get(id);
|
|
13655
14760
|
if (!item || reason !== null && !ACQUISITION_HOLD_REASONS.has(reason) ||
|
|
13656
14761
|
expected !== null && (!isRecord(expected) || !TAB_KEYS.includes(expected.resource) ||
|
|
13657
|
-
typeof expected.accessKey !== "string" || expected.accessKey.length === 0
|
|
14762
|
+
typeof expected.accessKey !== "string" || expected.accessKey.length === 0 ||
|
|
14763
|
+
expected.currentHold !== undefined &&
|
|
14764
|
+
(!isRecord(expected.currentHold) ||
|
|
14765
|
+
!ACQUISITION_HOLD_REASONS.has(expected.currentHold.reason) ||
|
|
14766
|
+
!Number.isFinite(expected.currentHold.at)))) {
|
|
13658
14767
|
return { ok: false, reason: "invalid" };
|
|
13659
14768
|
}
|
|
13660
14769
|
return transact((state) => {
|
|
@@ -13662,6 +14771,11 @@ function createAcquisitionEngine({
|
|
|
13662
14771
|
if (!record) return { ok: false, reason: "stale" };
|
|
13663
14772
|
if (expected && (record.query.resource !== expected.resource ||
|
|
13664
14773
|
record.query.accessKey !== expected.accessKey)) return { ok: false, reason: "stale-access" };
|
|
14774
|
+
if (expected?.currentHold !== undefined &&
|
|
14775
|
+
(record.hold?.reason !== expected.currentHold.reason ||
|
|
14776
|
+
record.hold?.at !== expected.currentHold.at)) {
|
|
14777
|
+
return { ok: false, reason: "stale-hold" };
|
|
14778
|
+
}
|
|
13665
14779
|
if ((record.hold?.reason ?? null) === reason) return { changed: false, value: true };
|
|
13666
14780
|
const next = reason === null ? null : { reason, at: now() };
|
|
13667
14781
|
record.hold = next;
|
|
@@ -13693,28 +14807,10 @@ function createAcquisitionEngine({
|
|
|
13693
14807
|
if (now() >= acquisitionClaimDeadline(record.claim)) {
|
|
13694
14808
|
return { ok: false, reason: "stale" };
|
|
13695
14809
|
}
|
|
13696
|
-
const receiptIds = [];
|
|
13697
|
-
for (const resource of RATE_RESOURCES) {
|
|
13698
|
-
if (costs[resource] <= 0) continue;
|
|
13699
|
-
const epoch = typeof supplied?.epochs?.[resource] === "string" && supplied.epochs[resource].length > 0
|
|
13700
|
-
? supplied.epochs[resource] : "unknown";
|
|
13701
|
-
const receipt = {
|
|
13702
|
-
id: `${reservationId}:${resource}`,
|
|
13703
|
-
reservationId,
|
|
13704
|
-
accessKey: record.query.accessKey,
|
|
13705
|
-
resource,
|
|
13706
|
-
epoch,
|
|
13707
|
-
units: costs[resource],
|
|
13708
|
-
startedAt: now(),
|
|
13709
|
-
};
|
|
13710
|
-
if (!addAcquisitionReceipt(state, receipt)) return { ok: false, reason: "capacity" };
|
|
13711
|
-
receiptIds.push(receipt.id);
|
|
13712
|
-
}
|
|
13713
14810
|
record.claim.started = true;
|
|
13714
14811
|
record.claim.startedAt = now();
|
|
13715
14812
|
record.claim.reservationId = reservationId;
|
|
13716
14813
|
record.claim.startReceipt = startReceipt;
|
|
13717
|
-
record.claim.receiptIds = receiptIds;
|
|
13718
14814
|
return { value: { status: "started", reservationId } };
|
|
13719
14815
|
});
|
|
13720
14816
|
}
|
|
@@ -13764,7 +14860,8 @@ function createAcquisitionEngine({
|
|
|
13764
14860
|
const claimed = transact((state) => {
|
|
13765
14861
|
const reaped = reapAcquisitionSubscriptions(state, now(), kill);
|
|
13766
14862
|
const subscription = state.subscriptions[id];
|
|
13767
|
-
const
|
|
14863
|
+
const queryKey = subscription?.queryKey;
|
|
14864
|
+
const record = state.queries[queryKey];
|
|
13768
14865
|
if (!subscription || subscription.nonce !== item.nonce || !record) return { ok: false, reason: "stale" };
|
|
13769
14866
|
const requestedGeneration = force
|
|
13770
14867
|
? Math.max(subscription.requestedGeneration,
|
|
@@ -13776,8 +14873,8 @@ function createAcquisitionEngine({
|
|
|
13776
14873
|
}
|
|
13777
14874
|
if (requestedGeneration <= record.generation && record.snapshot?.nextDueAt > now()) {
|
|
13778
14875
|
return { changed: reaped > 0 || demandChanged,
|
|
13779
|
-
value: { role: "follower", reason: "fresh", snapshot: record.snapshot,
|
|
13780
|
-
sharingCount: acquisitionSharingCount(state,
|
|
14876
|
+
value: { role: "follower", reason: "fresh", queryKey, snapshot: record.snapshot,
|
|
14877
|
+
sharingCount: acquisitionSharingCount(state, queryKey) } };
|
|
13781
14878
|
}
|
|
13782
14879
|
if (record.claim) {
|
|
13783
14880
|
const absoluteDeadline = acquisitionClaimDeadline(record.claim);
|
|
@@ -13792,10 +14889,10 @@ function createAcquisitionEngine({
|
|
|
13792
14889
|
}
|
|
13793
14890
|
return { changed: reaped > 0 || demandChanged || joined,
|
|
13794
14891
|
value: { role: "follower", reason: expired ? `owner-${owner}` : "claimed",
|
|
13795
|
-
snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state,
|
|
14892
|
+
queryKey, snapshot: record.snapshot, sharingCount: acquisitionSharingCount(state, queryKey) } };
|
|
13796
14893
|
}
|
|
13797
14894
|
}
|
|
13798
|
-
const nonce = createId("acquisition-claim",
|
|
14895
|
+
const nonce = createId("acquisition-claim", queryKey, record.generation + 1);
|
|
13799
14896
|
record.claim = {
|
|
13800
14897
|
pid,
|
|
13801
14898
|
nonce,
|
|
@@ -13809,11 +14906,13 @@ function createAcquisitionEngine({
|
|
|
13809
14906
|
subscription.waitingSinceAt = null;
|
|
13810
14907
|
record.lastUsedAt = now();
|
|
13811
14908
|
return { value: { role: "producer", nonce, generation: record.claim.generation,
|
|
13812
|
-
query: record.query, snapshot: record.snapshot,
|
|
13813
|
-
|
|
14909
|
+
queryKey, query: record.query, snapshot: record.snapshot,
|
|
14910
|
+
demand: aggregateAcquisitionDemand(state, queryKey),
|
|
14911
|
+
sharingCount: acquisitionSharingCount(state, queryKey) } };
|
|
13814
14912
|
});
|
|
14913
|
+
if (claimed.ok && claimed.value?.queryKey) item.queryKey = claimed.value.queryKey;
|
|
13815
14914
|
if (claimed.ok && claimed.value?.snapshot) {
|
|
13816
|
-
const record = claimed.state.queries[
|
|
14915
|
+
const record = claimed.state.queries[claimed.value.queryKey];
|
|
13817
14916
|
const loadedSnapshot = loadRecordSnapshot(record);
|
|
13818
14917
|
if (!loadedSnapshot.ok) return loadedSnapshot;
|
|
13819
14918
|
claimed.value.snapshot = loadedSnapshot.value;
|
|
@@ -13921,21 +15020,58 @@ function createAcquisitionEngine({
|
|
|
13921
15020
|
});
|
|
13922
15021
|
}
|
|
13923
15022
|
|
|
13924
|
-
function
|
|
13925
|
-
if (
|
|
15023
|
+
function syncQuotaProjection(scope) {
|
|
15024
|
+
if (!isRecord(scope) || !/^[0-9a-f]{64}$/.test(scope.hash ?? "")) {
|
|
13926
15025
|
return { ok: false, reason: "invalid" };
|
|
13927
15026
|
}
|
|
15027
|
+
// Quota is read before the acquisition lock. This diagnostic copy never
|
|
15028
|
+
// grants requests and a failed read cannot block source publication.
|
|
15029
|
+
const observed = inspectGovernor(scope, now());
|
|
15030
|
+
if (!observed.ok) {
|
|
15031
|
+
return transact((state) => {
|
|
15032
|
+
const previous = state.quotaProjections[scope.hash];
|
|
15033
|
+
if (!previous || previous.status === "unavailable") return { changed: false, value: false };
|
|
15034
|
+
previous.status = "unavailable";
|
|
15035
|
+
return { value: false };
|
|
15036
|
+
});
|
|
15037
|
+
}
|
|
15038
|
+
const ledger = observed.value;
|
|
15039
|
+
const units = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource,
|
|
15040
|
+
Object.values(ledger.reservations).reduce((sum, reservation) =>
|
|
15041
|
+
sum + reservationCost(reservation, resource, ledger.leases, now()),
|
|
15042
|
+
ledger.debt[resource].unresolvedUnits + ledger.debt[resource].quiescentUnits +
|
|
15043
|
+
reservationCost(ledger.controlReceipts[resource], resource, ledger.leases, now()))]));
|
|
15044
|
+
if (Object.values(units).some((value) => !Number.isSafeInteger(value) || value < 0)) {
|
|
15045
|
+
return { ok: false, reason: "accounting-overflow" };
|
|
15046
|
+
}
|
|
13928
15047
|
return transact((state) => {
|
|
13929
|
-
|
|
13930
|
-
|
|
13931
|
-
|
|
13932
|
-
|
|
13933
|
-
|
|
13934
|
-
|
|
13935
|
-
|
|
13936
|
-
|
|
15048
|
+
const activeKeys = new Set(Object.values(state.subscriptions)
|
|
15049
|
+
.map((subscription) => state.queries[subscription.queryKey]?.query.accessKey)
|
|
15050
|
+
.filter(Boolean));
|
|
15051
|
+
const currentAccessKey = /^[0-9a-f]{64}$/.test(scope.accessKey ?? "")
|
|
15052
|
+
? scope.accessKey : null;
|
|
15053
|
+
if (currentAccessKey) {
|
|
15054
|
+
for (const [hash, projection] of Object.entries(state.quotaProjections)) {
|
|
15055
|
+
if (hash !== scope.hash) projection.accessKeys = projection.accessKeys.filter((key) =>
|
|
15056
|
+
key !== currentAccessKey);
|
|
15057
|
+
}
|
|
13937
15058
|
}
|
|
13938
|
-
|
|
15059
|
+
const previous = state.quotaProjections[scope.hash];
|
|
15060
|
+
if (previous && previous.revision > ledger.revision) return { changed: false, value: false };
|
|
15061
|
+
if (!previous && Object.keys(state.quotaProjections).length >= ACQUISITION_MAX_QUOTA_PROJECTIONS) {
|
|
15062
|
+
const oldest = Object.entries(state.quotaProjections)
|
|
15063
|
+
.filter(([, projection]) => !projection.accessKeys.some((key) => activeKeys.has(key)))
|
|
15064
|
+
.sort((a, b) => a[1].at - b[1].at)[0];
|
|
15065
|
+
if (!oldest) return { changed: false, value: false };
|
|
15066
|
+
delete state.quotaProjections[oldest[0]];
|
|
15067
|
+
}
|
|
15068
|
+
const accessKeys = previous?.accessKeys.filter((key) => activeKeys.has(key)) ?? [];
|
|
15069
|
+
if (currentAccessKey && !accessKeys.includes(currentAccessKey) &&
|
|
15070
|
+
accessKeys.length < ACQUISITION_MAX_SUBSCRIPTIONS) accessKeys.push(currentAccessKey);
|
|
15071
|
+
state.quotaProjections[scope.hash] = {
|
|
15072
|
+
revision: ledger.revision, units, at: now(), status: "current", accessKeys,
|
|
15073
|
+
};
|
|
15074
|
+
return { value: true };
|
|
13939
15075
|
});
|
|
13940
15076
|
}
|
|
13941
15077
|
|
|
@@ -13948,7 +15084,7 @@ function createAcquisitionEngine({
|
|
|
13948
15084
|
inspect,
|
|
13949
15085
|
diagnostics,
|
|
13950
15086
|
recordMetrics,
|
|
13951
|
-
|
|
15087
|
+
syncQuotaProjection,
|
|
13952
15088
|
setHold,
|
|
13953
15089
|
close,
|
|
13954
15090
|
path,
|
|
@@ -13964,6 +15100,27 @@ function setRuntimeAcquisitionHold(engine, subscriptions, resource, reason, acce
|
|
|
13964
15100
|
return engine.setHold(subscription.id, reason, { resource, accessKey });
|
|
13965
15101
|
}
|
|
13966
15102
|
|
|
15103
|
+
function clearScheduledRuntimeControlHold(engine, subscriptions, resource, accessKey, observedAt) {
|
|
15104
|
+
const subscription = subscriptions?.get(resource);
|
|
15105
|
+
if (!subscription?.id) return { ok: false, reason: "stale" };
|
|
15106
|
+
if (typeof accessKey !== "string" || !Number.isFinite(observedAt)) {
|
|
15107
|
+
return { ok: false, reason: "stale-access" };
|
|
15108
|
+
}
|
|
15109
|
+
const inspected = engine.inspect(subscription.id);
|
|
15110
|
+
if (!inspected.ok) return inspected;
|
|
15111
|
+
const record = inspected.value;
|
|
15112
|
+
if (record?.query?.resource !== resource || record.query.accessKey !== accessKey) {
|
|
15113
|
+
return { ok: false, reason: "stale-access" };
|
|
15114
|
+
}
|
|
15115
|
+
const hold = record.hold;
|
|
15116
|
+
if (!hold) return { ok: true, value: "already-clear" };
|
|
15117
|
+
if (hold.reason !== "coordination") return { ok: true, value: "retained" };
|
|
15118
|
+
if (hold.at >= observedAt) return { ok: false, reason: "newer-hold" };
|
|
15119
|
+
const cleared = engine.setHold(subscription.id, null,
|
|
15120
|
+
{ resource, accessKey, currentHold: hold });
|
|
15121
|
+
return cleared.ok ? { ok: true, value: "cleared" } : cleared;
|
|
15122
|
+
}
|
|
15123
|
+
|
|
13967
15124
|
// The one place the cadence table is decided. It used to be spread across a
|
|
13968
15125
|
// fixed active floor, a fixed four-floor background slot, and a Security-only
|
|
13969
15126
|
// unchanged rule -- three policies that could disagree, and did: Security
|
|
@@ -13977,7 +15134,10 @@ function pollPolicyInterval({
|
|
|
13977
15134
|
unchangedCount = 0,
|
|
13978
15135
|
inProgressCI = false,
|
|
13979
15136
|
background = "all",
|
|
15137
|
+
requested = false,
|
|
13980
15138
|
}) {
|
|
15139
|
+
if (background === "on-demand" &&
|
|
15140
|
+
(demand !== "active" || (tab !== "actions" && !requested))) return Number.POSITIVE_INFINITY;
|
|
13981
15141
|
if (demand === "none") return Number.POSITIVE_INFINITY;
|
|
13982
15142
|
if (demand === "inactive") {
|
|
13983
15143
|
if (background === "off") return Number.POSITIVE_INFINITY;
|
|
@@ -14105,6 +15265,7 @@ function pollSchedule({
|
|
|
14105
15265
|
demand: isActive ? "active" : "inactive",
|
|
14106
15266
|
unchangedCount: states[key]?.unchangedCount ?? 0,
|
|
14107
15267
|
inProgressCI: states[key]?.inProgressCI === true,
|
|
15268
|
+
requested: states[key]?.requested === true,
|
|
14108
15269
|
background,
|
|
14109
15270
|
});
|
|
14110
15271
|
if (!Number.isFinite(step)) {
|
|
@@ -14249,7 +15410,7 @@ function governorDataReady(refreshResult, snapshot, activeKey, nowMs) {
|
|
|
14249
15410
|
return RATE_RESOURCES.every((resource) => {
|
|
14250
15411
|
if (costs[resource] <= 0) return true;
|
|
14251
15412
|
const budget = snapshot.value.budgets[resource];
|
|
14252
|
-
const available =
|
|
15413
|
+
const available = governorAvailableForGrant(snapshot.value, resource, nowMs);
|
|
14253
15414
|
return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource) &&
|
|
14254
15415
|
budget.blockUntil <= nowMs && available.mode === "open" &&
|
|
14255
15416
|
available.spendable >= costs[resource];
|
|
@@ -14456,6 +15617,335 @@ function coordinationNotice(reason) {
|
|
|
14456
15617
|
return "Can't coordinate API use — retrying";
|
|
14457
15618
|
}
|
|
14458
15619
|
|
|
15620
|
+
// Recovery diagnostics have no request payloads, credential identifiers or
|
|
15621
|
+
// owner capabilities. The journal is advisory: a failed write must never deny
|
|
15622
|
+
// an otherwise valid data operation. The quota ledger remains authoritative.
|
|
15623
|
+
const RECOVERY_JOURNAL_MAX_ENTRIES = 128;
|
|
15624
|
+
const RECOVERY_JOURNAL_MAX_BYTES = 128 * 1024;
|
|
15625
|
+
const RECOVERY_ENTRY_MAX_BYTES = 1024;
|
|
15626
|
+
const RECOVERY_COPY = Object.freeze({
|
|
15627
|
+
busy: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
|
|
15628
|
+
stale: ["Coordinating with your other panes", "Retry automatically", "Other pane; retry"],
|
|
15629
|
+
"identity-busy": ["GitHub budget check is shared", "Retry automatically after the current check", "Budget check; retry"],
|
|
15630
|
+
"transport-busy": ["Shared GitHub request is in progress", "Retry automatically", "Request; retry"],
|
|
15631
|
+
unwritable: ["Local coordination cannot be written", "Run gh-glance --doctor; check storage permissions", "Store blocked; doctor"],
|
|
15632
|
+
corrupt: ["Local coordination is corrupt", "Run gh-glance --doctor; preserve the reported state file", "State corrupt; doctor"],
|
|
15633
|
+
capacity: ["Local coordination is full", "Run gh-glance --doctor; preserve the reported state file", "State full; doctor"],
|
|
15634
|
+
"accounting-overflow": ["Local coordination accounting overflow", "Run gh-glance --doctor; preserve the reported state file", "Overflow; doctor"],
|
|
15635
|
+
"reservations-invalid": ["Local coordination reservation invalid", "Run gh-glance --doctor; preserve the reported state file", "Reservation; doctor"],
|
|
15636
|
+
"external-factor-invalid": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
|
|
15637
|
+
"pacing-invalid": ["Local coordination pacing invalid", "Run gh-glance --doctor; preserve the reported state file", "Pacing invalid; doctor"],
|
|
15638
|
+
"budget-resource": ["Local coordination budget invalid", "Run gh-glance --doctor; preserve the reported state file", "Budget invalid; doctor"],
|
|
15639
|
+
"legacy-unresolved": ["Older requests still reserve quota", "Verify old children ended; a verified system restart can resolve unknown ownership", "Old debt; restart"],
|
|
15640
|
+
"legacy-corrupt": ["Older coordination evidence is corrupt", "Run gh-glance --doctor; preserve the reported state file", "Old state; doctor"],
|
|
15641
|
+
"migration-hold": ["Older session still uses coordination", "Wait for the older quota reset; retry automatically at the shown time", "Old quota; wait"],
|
|
15642
|
+
"restart-required": ["Older session still uses coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
|
|
15643
|
+
"identity-backoff": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
|
|
15644
|
+
"identity-capacity": ["GitHub budget check allowance is full", "Retry automatically when the rolling window renews", "Budget check; retry"],
|
|
15645
|
+
"budget-unknown": ["GitHub budget check pending", "Retry automatically", "Budget check; retry"],
|
|
15646
|
+
"budget-stale": ["GitHub budget evidence is old", "Retry the shared budget check automatically", "Budget old; retry"],
|
|
15647
|
+
"budget-future": ["Clock changed; budget needs recheck", "Retry the shared budget check automatically", "Clock; retry"],
|
|
15648
|
+
"probe-failed": ["GitHub budget check failed", "Retry automatically at the shown time", "Budget check; retry"],
|
|
15649
|
+
"block-unpublished": ["Saving GitHub request limit", "Retry sharing the hold automatically", "Saving hold; retry"],
|
|
15650
|
+
"budget-reset": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
|
|
15651
|
+
reset: ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
|
|
15652
|
+
priority: ["Background refresh waiting its turn", "Retry automatically at the next slot", "Queued; retry"],
|
|
15653
|
+
"rate-limit": ["GitHub quota pacing", "Retry automatically after reset", "Quota; retry"],
|
|
15654
|
+
"rate-limited": ["GitHub request limit", "Retry automatically after the shared hold", "Limit; retry"],
|
|
15655
|
+
"local-reserve": ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
|
|
15656
|
+
reserve: ["Local quota reserve pacing", "Retry automatically at the next grant", "Reserve; retry"],
|
|
15657
|
+
blocked: ["GitHub request hold", "Retry automatically when the hold ends", "Request hold; retry"],
|
|
15658
|
+
secondary: ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
|
|
15659
|
+
"secondary-rate-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
|
|
15660
|
+
"abuse-limit": ["GitHub request limit", "Retry after GitHub's hold; refresh only if the pause asks", "Limit; retry"],
|
|
15661
|
+
"throttle-paused": ["Repeated GitHub request limit", "Press r to retry after checking the hold", "Limit; press r"],
|
|
15662
|
+
"network-outage": ["GitHub connection failed", "Retry automatically when the connection returns", "Network; retry"],
|
|
15663
|
+
"credential-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
|
|
15664
|
+
"identity-unavailable": ["GitHub login unavailable", "Run gh auth status, then restore login", "Login; gh auth status"],
|
|
15665
|
+
disconnected: ["Collector connection unavailable", "Restore the selected collector connection", "Collector; connection"],
|
|
15666
|
+
"app-auth": ["GitHub App authorization unavailable", "Restore the selected App authorization", "App; restore auth"],
|
|
15667
|
+
"security-incomplete": ["Security data incomplete", "Run gh-glance --doctor --probe for endpoint access", "Security; doctor"],
|
|
15668
|
+
"receipt-retry": ["Saving request result", "Retry automatically without repeating the request", "Saving; retry"],
|
|
15669
|
+
"interrupted-request": ["Recovering interrupted request", "Retry source observation when quota allows", "Interrupted; retry"],
|
|
15670
|
+
"request-queue": ["Recovering local request queue", "Retry automatically after compaction", "Queue; retry"],
|
|
15671
|
+
"acquisition-busy": ["Acquisition store is busy", "Check the shared lock; retry automatically", "Store lock; retry"],
|
|
15672
|
+
"clock-recovery": ["Rechecking after sleep or clock change", "Retry source observation automatically", "Clock; retry"],
|
|
15673
|
+
"auth-problem": ["GitHub authorization failed", "Run gh auth status, then restore authorization", "Auth; gh auth status"],
|
|
15674
|
+
unavailable: ["Repository or endpoint unavailable", "Check the selected repository and permissions", "Repo; check access"],
|
|
15675
|
+
"source-incomplete": ["GitHub response incomplete", "Retry source observation automatically", "Incomplete; retry"],
|
|
15676
|
+
"unknown-scope": ["GitHub account scope unavailable", "Run gh auth status; retry when identity is verified", "Login; gh auth status"],
|
|
15677
|
+
"version-mismatch": ["Older session cannot share coordination", "Restart older gh-glance panes with this version", "Old pane; restart"],
|
|
15678
|
+
"unsafe-permissions": ["Local coordination permissions are unsafe", "Run gh-glance --doctor; repair directory permissions", "Permissions; doctor"],
|
|
15679
|
+
"disk-full": ["Local coordination storage is full", "Run gh-glance --doctor; free local storage", "Disk full; doctor"],
|
|
15680
|
+
coordination: ["Local coordination unavailable", "Run gh-glance --doctor; retry when storage works", "Coordination; doctor"],
|
|
15681
|
+
});
|
|
15682
|
+
|
|
15683
|
+
function recoveryCause({ reason, resource = null, at = Date.now(), retryAt = null,
|
|
15684
|
+
sourceAt = null, receiptUnits = null, debtUnits = null, oldestUnresolvedAt = null,
|
|
15685
|
+
observerAt = null, terminalizationBacklog = null, lastFailedTransition = null,
|
|
15686
|
+
origin = "coordination", tab = null } = {}) {
|
|
15687
|
+
const code = typeof reason === "string" && Object.hasOwn(RECOVERY_COPY, reason)
|
|
15688
|
+
? reason : "coordination";
|
|
15689
|
+
const copy = RECOVERY_COPY[code] ?? RECOVERY_COPY.coordination;
|
|
15690
|
+
const time = (value) => Number.isFinite(value) && value >= 0 ? value : null;
|
|
15691
|
+
const units = (value) => Number.isSafeInteger(value) && value >= 0 ? value : null;
|
|
15692
|
+
return {
|
|
15693
|
+
code, resource: RATE_RESOURCES.includes(resource) ? resource : null,
|
|
15694
|
+
origin: origin === "source" ? "source" : "coordination",
|
|
15695
|
+
tab: TAB_KEYS.includes(tab) ? tab : null,
|
|
15696
|
+
at: time(at) ?? Date.now(), firstAt: time(at) ?? Date.now(),
|
|
15697
|
+
retryAt: time(retryAt), sourceAt: time(sourceAt),
|
|
15698
|
+
receiptUnits: units(receiptUnits), debtUnits: units(debtUnits),
|
|
15699
|
+
oldestUnresolvedAt: time(oldestUnresolvedAt), observerAt: time(observerAt),
|
|
15700
|
+
terminalizationBacklog: units(terminalizationBacklog),
|
|
15701
|
+
lastFailedTransition: typeof lastFailedTransition === "string" && Object.hasOwn(RECOVERY_COPY, lastFailedTransition)
|
|
15702
|
+
? lastFailedTransition : code,
|
|
15703
|
+
summary: copy[0], action: copy[1], short: copy[2],
|
|
15704
|
+
};
|
|
15705
|
+
}
|
|
15706
|
+
|
|
15707
|
+
function shouldClearRecoveryCause(cause, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
|
|
15708
|
+
if (!cause) return false;
|
|
15709
|
+
if (scheduledHold !== null) {
|
|
15710
|
+
return scheduledHold.ok === true && scheduledHold.value === "cleared" &&
|
|
15711
|
+
cause.origin === "coordination" && ["busy", "stale", "acquisition-busy"].includes(cause.code);
|
|
15712
|
+
}
|
|
15713
|
+
return cause.origin !== "source" || sourceSuccess && cause.tab === tab;
|
|
15714
|
+
}
|
|
15715
|
+
|
|
15716
|
+
function recoveryRetryAt(decision, budget, observer, nowMs) {
|
|
15717
|
+
if (budget?.blockUntil > nowMs) return budget.blockUntil;
|
|
15718
|
+
if (["reservations-invalid", "external-factor-invalid", "pacing-invalid", "budget-resource",
|
|
15719
|
+
"accounting-overflow", "corrupt", "unwritable", "capacity"].includes(decision?.reason)) return null;
|
|
15720
|
+
const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision?.reason);
|
|
15721
|
+
if (decision?.mode === "probe" && !resetHold) {
|
|
15722
|
+
return observer?.nextAt > nowMs ? observer.nextAt : null;
|
|
15723
|
+
}
|
|
15724
|
+
const deadline = decision?.retryAt ?? decision?.resetMs ?? budget?.resetMs;
|
|
15725
|
+
return deadline > nowMs ? deadline : null;
|
|
15726
|
+
}
|
|
15727
|
+
|
|
15728
|
+
function terminalizationRecoveryCode(terminal) {
|
|
15729
|
+
if (terminal?.status === "retryable") return "receipt-retry";
|
|
15730
|
+
if (terminal?.status === "compacted") return "interrupted-request";
|
|
15731
|
+
if (terminal?.reason === "completion-capacity") return "capacity";
|
|
15732
|
+
return terminal?.reason ?? terminal?.status ?? "coordination";
|
|
15733
|
+
}
|
|
15734
|
+
|
|
15735
|
+
function requestFailureRecoveryCode(error, verdict) {
|
|
15736
|
+
if (verdict === "rate-limited") return "rate-limited";
|
|
15737
|
+
if (Object.hasOwn(HTTP_PERMIT_FAILURE_HOLDS, error?.coordinationReason)) return error.coordinationReason;
|
|
15738
|
+
if (verdict === "other" && error?.httpStarted === true) return "interrupted-request";
|
|
15739
|
+
return verdict === "other" ? "network-outage" : verdict;
|
|
15740
|
+
}
|
|
15741
|
+
|
|
15742
|
+
function requestFailureRecovery(error, verdict) {
|
|
15743
|
+
const reason = requestFailureRecoveryCode(error, verdict);
|
|
15744
|
+
return { reason, origin: reason === "transport-busy" ? "coordination" : "source" };
|
|
15745
|
+
}
|
|
15746
|
+
|
|
15747
|
+
function wallClockRecovery(previousAt, at, intervalMs) {
|
|
15748
|
+
if (!Number.isFinite(previousAt) || !Number.isFinite(at)) return null;
|
|
15749
|
+
return at < previousAt || at - previousAt > Math.max(30_000, 3 * intervalMs)
|
|
15750
|
+
? "clock-recovery" : null;
|
|
15751
|
+
}
|
|
15752
|
+
|
|
15753
|
+
function collectorSourceAdvanced(previousAt, sourceAt) {
|
|
15754
|
+
return Number.isFinite(previousAt) && Number.isFinite(sourceAt) && sourceAt > previousAt;
|
|
15755
|
+
}
|
|
15756
|
+
|
|
15757
|
+
function cleanupQueueForResource(queue, resource) {
|
|
15758
|
+
let count = 0;
|
|
15759
|
+
for (const item of queue) if (tabRequestCost(item.cleanupKey)?.[resource] > 0) count += 1;
|
|
15760
|
+
return count;
|
|
15761
|
+
}
|
|
15762
|
+
|
|
15763
|
+
function presentRecovery(cause, { nowMs = Date.now(), cols = 80 } = {}) {
|
|
15764
|
+
if (!cause) return [];
|
|
15765
|
+
const width = Math.max(1, Number.isSafeInteger(cols) ? cols - 1 : 79);
|
|
15766
|
+
const resource = cause.resource === "graphql" ? "GraphQL" : cause.resource === "core" ? "Core" : "GitHub";
|
|
15767
|
+
const age = Number.isFinite(cause.sourceAt) && cause.sourceAt <= nowMs
|
|
15768
|
+
? `cached ${formatDuration(nowMs - cause.sourceAt)}` : "cached age unavailable";
|
|
15769
|
+
const retry = Number.isFinite(cause.retryAt)
|
|
15770
|
+
? `; next ${statusInterval(cause.retryAt, nowMs)}` : "";
|
|
15771
|
+
const summary = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
|
|
15772
|
+
? `Older requests still reserve ${cause.debtUnits} units` : cause.summary;
|
|
15773
|
+
if (width < 60) {
|
|
15774
|
+
const action = cause.action.toLowerCase();
|
|
15775
|
+
const narrowAction = cause.code === "legacy-unresolved" ? "Check children; restart"
|
|
15776
|
+
: action.includes("gh-glance --doctor") ? "Run gh-glance --doctor"
|
|
15777
|
+
: action.includes("gh auth") ? "Run gh auth status"
|
|
15778
|
+
: action.includes("selected repository") ? "Check repository access"
|
|
15779
|
+
: action.includes("connection") ? "Restore connection"
|
|
15780
|
+
: action.includes("app authorization") ? "Restore App auth"
|
|
15781
|
+
: action.includes("permissions") ? "Check permissions"
|
|
15782
|
+
: action.includes("restart") ? "Restart and recheck"
|
|
15783
|
+
: action.includes("press r") ? "Press r to retry"
|
|
15784
|
+
: "Retry automatically";
|
|
15785
|
+
const narrowDeadline = Number.isFinite(cause.retryAt)
|
|
15786
|
+
? statusInterval(cause.retryAt, nowMs) : null;
|
|
15787
|
+
const timedAction = narrowDeadline && cause.code === "migration-hold"
|
|
15788
|
+
? `Wait; next ${narrowDeadline}`
|
|
15789
|
+
: narrowDeadline && narrowAction === "Retry automatically"
|
|
15790
|
+
? `Retry; next ${narrowDeadline}`
|
|
15791
|
+
: narrowDeadline && narrowAction === "Press r to retry"
|
|
15792
|
+
? `Press r after ${narrowDeadline}` : narrowAction;
|
|
15793
|
+
const short = cause.code === "legacy-unresolved" && Number.isSafeInteger(cause.debtUnits)
|
|
15794
|
+
? `Debt ${cause.debtUnits} units` : cause.short.split(";")[0];
|
|
15795
|
+
return [`${resource}: ${short}`.slice(0, width), timedAction.slice(0, width), age.slice(0, width)];
|
|
15796
|
+
}
|
|
15797
|
+
const full = `${resource}: ${summary}; ${cause.action}${retry}; ${age}`;
|
|
15798
|
+
if (full.length <= width) return [full];
|
|
15799
|
+
return [`${resource}: ${summary}`.slice(0, width),
|
|
15800
|
+
`${cause.action}${retry}`.slice(0, width), age.slice(0, width)];
|
|
15801
|
+
}
|
|
15802
|
+
|
|
15803
|
+
function recoveryJournalPath(scope) {
|
|
15804
|
+
return `${scope.path}.recovery.json`;
|
|
15805
|
+
}
|
|
15806
|
+
|
|
15807
|
+
function emptyRecoveryJournal() {
|
|
15808
|
+
return { version: 1, current: { core: null, graphql: null }, entries: [] };
|
|
15809
|
+
}
|
|
15810
|
+
|
|
15811
|
+
const RECOVERY_EVENT_KEYS = ["code", "resource", "at", "firstAt", "retryAt", "sourceAt",
|
|
15812
|
+
"receiptUnits", "debtUnits", "oldestUnresolvedAt", "observerAt", "terminalizationBacklog",
|
|
15813
|
+
"lastFailedTransition", "origin", "tab"];
|
|
15814
|
+
|
|
15815
|
+
function recoveryEvent(entry) {
|
|
15816
|
+
if (!isRecord(entry)) return null;
|
|
15817
|
+
if (exactKeys(entry, ["code", "resource", "at"]) && entry.code === "clear" &&
|
|
15818
|
+
RATE_RESOURCES.includes(entry.resource) && Number.isFinite(entry.at)) return entry;
|
|
15819
|
+
// An earlier clock sample can arrive after a later one for the same cause.
|
|
15820
|
+
// Repair only this bounded timestamp skew; every other field stays subject
|
|
15821
|
+
// to the strict journal validation below.
|
|
15822
|
+
const event = Number.isFinite(entry.at) && Number.isFinite(entry.firstAt) && entry.firstAt > entry.at
|
|
15823
|
+
? { ...entry, firstAt: entry.at } : entry;
|
|
15824
|
+
if (!exactKeys(event, RECOVERY_EVENT_KEYS) || !RATE_RESOURCES.includes(event.resource) ||
|
|
15825
|
+
typeof event.code !== "string" || !Object.hasOwn(RECOVERY_COPY, event.code) ||
|
|
15826
|
+
!Number.isFinite(event.at) || !Number.isFinite(event.firstAt) ||
|
|
15827
|
+
["retryAt", "sourceAt", "oldestUnresolvedAt", "observerAt"].some((key) =>
|
|
15828
|
+
event[key] !== null && (!Number.isFinite(event[key]) || event[key] < 0)) ||
|
|
15829
|
+
["receiptUnits", "debtUnits", "terminalizationBacklog"].some((key) =>
|
|
15830
|
+
event[key] !== null && (!Number.isSafeInteger(event[key]) || event[key] < 0)) ||
|
|
15831
|
+
typeof event.lastFailedTransition !== "string" ||
|
|
15832
|
+
!Object.hasOwn(RECOVERY_COPY, event.lastFailedTransition) ||
|
|
15833
|
+
!["source", "coordination"].includes(event.origin) ||
|
|
15834
|
+
(event.tab !== null && !TAB_KEYS.includes(event.tab)) ||
|
|
15835
|
+
Buffer.byteLength(JSON.stringify(event)) > RECOVERY_ENTRY_MAX_BYTES) return null;
|
|
15836
|
+
return event;
|
|
15837
|
+
}
|
|
15838
|
+
|
|
15839
|
+
function visibleRecoveryEvent(entry) {
|
|
15840
|
+
if (!entry || entry.code === "clear") return entry;
|
|
15841
|
+
const copy = RECOVERY_COPY[entry.code] ?? RECOVERY_COPY.coordination;
|
|
15842
|
+
return { ...entry, summary: copy[0], action: copy[1], short: copy[2] };
|
|
15843
|
+
}
|
|
15844
|
+
|
|
15845
|
+
function readRecoveryTransitions(scope) {
|
|
15846
|
+
const path = recoveryJournalPath(scope);
|
|
15847
|
+
let raw;
|
|
15848
|
+
try {
|
|
15849
|
+
const file = lstatSync(path);
|
|
15850
|
+
if (!file.isFile() || file.size > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
|
|
15851
|
+
raw = readFileSync(path, "utf8");
|
|
15852
|
+
if (Buffer.byteLength(raw) > RECOVERY_JOURNAL_MAX_BYTES) return { ...emptyRecoveryJournal(), status: "unavailable" };
|
|
15853
|
+
} catch (error) {
|
|
15854
|
+
return { ...emptyRecoveryJournal(), status: error?.code === "ENOENT" ? "missing" : "unavailable" };
|
|
15855
|
+
}
|
|
15856
|
+
try {
|
|
15857
|
+
const journal = JSON.parse(raw);
|
|
15858
|
+
const entries = journal.entries?.map(recoveryEvent);
|
|
15859
|
+
const current = journal.current && Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
15860
|
+
resource, journal.current[resource] === null ? null : recoveryEvent(journal.current[resource]),
|
|
15861
|
+
]));
|
|
15862
|
+
if (!exactKeys(journal, ["version", "current", "entries"]) || journal.version !== 1 ||
|
|
15863
|
+
!exactKeys(journal.current, RATE_RESOURCES) || !Array.isArray(journal.entries) ||
|
|
15864
|
+
journal.entries.length > RECOVERY_JOURNAL_MAX_ENTRIES ||
|
|
15865
|
+
entries.some((entry) => entry === null) ||
|
|
15866
|
+
RATE_RESOURCES.some((resource) => journal.current[resource] !== null &&
|
|
15867
|
+
(current[resource] === null || current[resource].code === "clear"))) {
|
|
15868
|
+
return { ...emptyRecoveryJournal(), status: "corrupt" };
|
|
15869
|
+
}
|
|
15870
|
+
return { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
|
|
15871
|
+
[resource, visibleRecoveryEvent(current[resource])])),
|
|
15872
|
+
entries: entries.map(visibleRecoveryEvent), status: "healthy" };
|
|
15873
|
+
} catch {
|
|
15874
|
+
return { ...emptyRecoveryJournal(), status: "corrupt" };
|
|
15875
|
+
}
|
|
15876
|
+
}
|
|
15877
|
+
|
|
15878
|
+
function recordRecoveryTransition(scope, event) {
|
|
15879
|
+
if (!scope?.path || !RATE_RESOURCES.includes(event?.resource)) return { ok: false, reason: "invalid" };
|
|
15880
|
+
const path = recoveryJournalPath(scope);
|
|
15881
|
+
return withPersistenceLock(path, () => {
|
|
15882
|
+
const previous = readRecoveryTransitions(scope);
|
|
15883
|
+
if (!["healthy", "missing"].includes(previous.status)) return { ok: false, reason: previous.status };
|
|
15884
|
+
const plain = (entry) => entry && Object.fromEntries(RECOVERY_EVENT_KEYS.map((key) => [key, entry[key]]));
|
|
15885
|
+
const journal = { version: 1, current: Object.fromEntries(RATE_RESOURCES.map((resource) =>
|
|
15886
|
+
[resource, plain(previous.current[resource])])),
|
|
15887
|
+
entries: previous.entries.map((entry) => entry.code === "clear" ? entry : plain(entry)) };
|
|
15888
|
+
const former = journal.current[event.resource];
|
|
15889
|
+
const at = Number.isFinite(event.at) ? event.at : Date.now();
|
|
15890
|
+
if (event.clear === true) {
|
|
15891
|
+
if (!former) return { ok: true, changed: false };
|
|
15892
|
+
journal.current[event.resource] = null;
|
|
15893
|
+
journal.entries.push({ code: "clear", resource: event.resource, at });
|
|
15894
|
+
} else {
|
|
15895
|
+
const entry = plain(recoveryCause({ ...event, reason: event.reason ?? event.code, at }));
|
|
15896
|
+
entry.firstAt = former?.code === entry.code && former?.origin === entry.origin &&
|
|
15897
|
+
former?.tab === entry.tab ? Math.min(former.firstAt, at) : at;
|
|
15898
|
+
if (Buffer.byteLength(JSON.stringify(entry)) > RECOVERY_ENTRY_MAX_BYTES) return { ok: false, reason: "capacity" };
|
|
15899
|
+
journal.current[event.resource] = entry;
|
|
15900
|
+
if (former?.code !== entry.code || former?.origin !== entry.origin || former?.tab !== entry.tab) {
|
|
15901
|
+
journal.entries.push(entry);
|
|
15902
|
+
}
|
|
15903
|
+
}
|
|
15904
|
+
journal.entries = journal.entries.slice(-RECOVERY_JOURNAL_MAX_ENTRIES);
|
|
15905
|
+
while (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES && journal.entries.length > 0) {
|
|
15906
|
+
journal.entries.shift();
|
|
15907
|
+
}
|
|
15908
|
+
if (Buffer.byteLength(JSON.stringify(journal)) + 1 > RECOVERY_JOURNAL_MAX_BYTES) return { ok: false, reason: "capacity" };
|
|
15909
|
+
const temp = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
15910
|
+
try {
|
|
15911
|
+
writeFileSync(temp, `${JSON.stringify(journal)}\n`, { mode: 0o600, flag: "wx" });
|
|
15912
|
+
renameSync(temp, path);
|
|
15913
|
+
return { ok: true, changed: true };
|
|
15914
|
+
} catch (error) {
|
|
15915
|
+
try { unlinkSync(temp); } catch { /* only our temporary file */ }
|
|
15916
|
+
return { ok: false, reason: error?.code ?? "unwritable" };
|
|
15917
|
+
}
|
|
15918
|
+
});
|
|
15919
|
+
}
|
|
15920
|
+
|
|
15921
|
+
function safeRecordRecoveryTransition(scope, event, write = recordRecoveryTransition) {
|
|
15922
|
+
try { return write(scope, event); } catch { return { ok: false, reason: "unavailable" }; }
|
|
15923
|
+
}
|
|
15924
|
+
|
|
15925
|
+
function recoveryLedgerSummary(scope, resource, nowMs = Date.now()) {
|
|
15926
|
+
if (!scope?.path || !RATE_RESOURCES.includes(resource)) return null;
|
|
15927
|
+
const loaded = readGovernorState(scope.path, nowMs, { persistMigration: false });
|
|
15928
|
+
if (!loaded.ok) return null;
|
|
15929
|
+
const state = loaded.value;
|
|
15930
|
+
const debt = state.debt[resource];
|
|
15931
|
+
const debtUnits = debt.unresolvedUnits + debt.quiescentUnits;
|
|
15932
|
+
const charged = governorChargedCost(state, resource, nowMs);
|
|
15933
|
+
const oldest = Object.values(state.debtGroups)
|
|
15934
|
+
.filter((group) => !group.quiescent && group.units[resource] > 0)
|
|
15935
|
+
.map((group) => group.at);
|
|
15936
|
+
return {
|
|
15937
|
+
receiptUnits: Number.isSafeInteger(charged) ? charged - debtUnits : null,
|
|
15938
|
+
debtUnits,
|
|
15939
|
+
oldestUnresolvedAt: oldest.length > 0 ? Math.min(...oldest) : null,
|
|
15940
|
+
observerAt: state.observers[resource]?.at > 0 && state.observers[resource].at <= nowMs
|
|
15941
|
+
? state.observers[resource].at : null,
|
|
15942
|
+
debtCount: debt.unresolvedCount + debt.quiescentCount,
|
|
15943
|
+
receiptCount: Object.values(state.reservations).filter((reservation) =>
|
|
15944
|
+
reservationCost(reservation, resource, state.leases, nowMs) > 0).length +
|
|
15945
|
+
(reservationCost(state.controlReceipts[resource], resource, state.leases, nowMs) > 0 ? 1 : 0),
|
|
15946
|
+
};
|
|
15947
|
+
}
|
|
15948
|
+
|
|
14459
15949
|
function acquisitionFailureCopy(reason) {
|
|
14460
15950
|
if (reason === "busy") return { short: "lock busy", narrow: "Shared lock busy", full: "Acquisition store is busy; checking the shared lock" };
|
|
14461
15951
|
if (reason === "unwritable") return { short: "store blocked", narrow: "Store blocked", full: "Acquisition store cannot be written; check config permissions" };
|
|
@@ -14557,25 +16047,6 @@ function measuredNestedOperationCost(operation, value) {
|
|
|
14557
16047
|
return actual;
|
|
14558
16048
|
}
|
|
14559
16049
|
|
|
14560
|
-
function uncertainReservationReceipts(scope, reservationId, nowMs) {
|
|
14561
|
-
const snapshot = inspectGovernor(scope, nowMs);
|
|
14562
|
-
const reservation = snapshot.ok ? snapshot.value.reservations?.[reservationId] : null;
|
|
14563
|
-
const accessKey = scope.accessKey ?? scope.hash;
|
|
14564
|
-
if (!reservation || typeof accessKey !== "string") return [];
|
|
14565
|
-
return RATE_RESOURCES.flatMap((resource) => {
|
|
14566
|
-
if (reservation.costs[resource] <= 0) return [];
|
|
14567
|
-
return [{
|
|
14568
|
-
id: `${reservationId}:${resource}`,
|
|
14569
|
-
reservationId,
|
|
14570
|
-
accessKey,
|
|
14571
|
-
resource,
|
|
14572
|
-
epoch: reservation.epochs[resource],
|
|
14573
|
-
units: reservation.costs[resource],
|
|
14574
|
-
startedAt: reservation.startedAt,
|
|
14575
|
-
}];
|
|
14576
|
-
});
|
|
14577
|
-
}
|
|
14578
|
-
|
|
14579
16050
|
function abortableDelay(ms, signal, {
|
|
14580
16051
|
setTimeout: setTimeout_ = setTimeout,
|
|
14581
16052
|
clearTimeout: clearTimeout_ = clearTimeout,
|
|
@@ -14770,30 +16241,38 @@ async function runAdmittedOperation({
|
|
|
14770
16241
|
}
|
|
14771
16242
|
const reservationId = admitted.value.reservationId;
|
|
14772
16243
|
const settlementScope = { ...scope, identityProvider: null };
|
|
14773
|
-
const
|
|
16244
|
+
const transportScope = { ...scope, receiptCapability: admitted.value.receiptCapability };
|
|
14774
16245
|
try {
|
|
14775
|
-
const value = await requestIdentityStorage.run(
|
|
16246
|
+
const value = await requestIdentityStorage.run(transportScope, () => run(signal));
|
|
14776
16247
|
const measured = measuredNestedOperationCost(operation, value);
|
|
14777
16248
|
const costs = measured ?? operationCost(operation);
|
|
14778
|
-
const outcome = value?.
|
|
14779
|
-
|
|
16249
|
+
const outcome = value?.notStarted === true ? "measured-success" :
|
|
16250
|
+
value?.ok === false ? governorOutcomeForError(value.failure) : "measured-success";
|
|
16251
|
+
const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
|
|
14780
16252
|
outcome,
|
|
14781
|
-
...(
|
|
14782
|
-
|
|
16253
|
+
...(value?.notStarted === true ? { neverStarted: true } : {}),
|
|
16254
|
+
...(measured || outcome === "measured-success" ? { actualCosts: costs } : {}),
|
|
16255
|
+
}, now);
|
|
16256
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
16257
|
+
return { ok: false, error: new Error(`API receipt settlement ${terminal.status}`),
|
|
16258
|
+
reservationId };
|
|
16259
|
+
}
|
|
14783
16260
|
const identityCoordinator = scope.identityCoordinator ?? runtimeIdentityCoordinator;
|
|
14784
16261
|
if (scope.accessKey && scope.accessKey !== identityCoordinator?.current()?.accessKey) {
|
|
14785
16262
|
return { ok: false, error: new Error("Credential changed"), reservationId };
|
|
14786
16263
|
}
|
|
14787
|
-
return { ok: true, value, reservationId
|
|
14788
|
-
uncertainReceipts: measured ? [] : pendingReceipts };
|
|
16264
|
+
return { ok: true, value, reservationId };
|
|
14789
16265
|
} catch (error) {
|
|
14790
16266
|
const actualCost = measuredNestedOperationCost(operation, { requestMetrics: error?.requestMetrics });
|
|
14791
|
-
const
|
|
14792
|
-
completeReservation(settlementScope, reservationId, {
|
|
16267
|
+
const terminal = queueTerminalization(settlementScope, admitted.value.receiptCapability, {
|
|
14793
16268
|
outcome: governorOutcomeForError(error),
|
|
14794
|
-
...(
|
|
14795
|
-
|
|
14796
|
-
|
|
16269
|
+
...(error?.notStarted === true ? { neverStarted: true } : {}),
|
|
16270
|
+
...(actualCost ? { actualCosts: actualCost } : {}),
|
|
16271
|
+
}, now);
|
|
16272
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
16273
|
+
error.receiptSettlement = terminal.status;
|
|
16274
|
+
}
|
|
16275
|
+
return { ok: false, error, reservationId };
|
|
14797
16276
|
}
|
|
14798
16277
|
}
|
|
14799
16278
|
|
|
@@ -15010,6 +16489,7 @@ function refreshStatus({
|
|
|
15010
16489
|
stale = false,
|
|
15011
16490
|
disconnected = false,
|
|
15012
16491
|
screenReader = false,
|
|
16492
|
+
onDemand = false,
|
|
15013
16493
|
} = {}) {
|
|
15014
16494
|
const status = (kind, glyphKind, label, tone, animate = false, detailKind = null) => ({
|
|
15015
16495
|
kind,
|
|
@@ -15046,6 +16526,7 @@ function refreshStatus({
|
|
|
15046
16526
|
// Keep the cause visible until a successful result clears it.
|
|
15047
16527
|
if (activeError) return status("failed", "failed", "Failed", "attention");
|
|
15048
16528
|
if (securityIncomplete) return status("limited", "limited", "Limited", "attention");
|
|
16529
|
+
if (onDemand) return status("cached", "watching", "Cached", "inert");
|
|
15049
16530
|
if (sharedData) return status("shared", "watching", "Shared", "inert", false, "sharing");
|
|
15050
16531
|
if (["waiting", "pending", "probe"].includes(mode)) {
|
|
15051
16532
|
const sharing = sharedLaneProvenance(governorDecision).waitCause === "shared-lane";
|
|
@@ -15535,6 +17016,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15535
17016
|
const [waiting, setWaiting] = useState({ actions: true, issues: true, prs: true, security: true });
|
|
15536
17017
|
const [governorDecisions, setGovernorDecisions] = useState(probingGovernorDecisions);
|
|
15537
17018
|
const [acquisitionFailures, setAcquisitionFailures] = useState({});
|
|
17019
|
+
const [recoveryCauses, setRecoveryCauses] = useState({ core: null, graphql: null });
|
|
17020
|
+
const recoveryRef = useRef(recoveryCauses);
|
|
15538
17021
|
const [governorEpochs, setGovernorEpochs] = useState(null);
|
|
15539
17022
|
const [requestStatuses, setRequestStatuses] = useState(() =>
|
|
15540
17023
|
Object.fromEntries(TABS.map((candidate) => [candidate.key, null])),
|
|
@@ -15560,6 +17043,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15560
17043
|
? { mode: "paused", reason: activeFailure, coordinationError: true,
|
|
15561
17044
|
cause: acquisitionFailureCopy(activeFailure).short }
|
|
15562
17045
|
: governorDecisions[tab.key];
|
|
17046
|
+
const activeResource = RATE_RESOURCES.find((resource) => tabRequestCost(tab.key)[resource] > 0) ?? null;
|
|
17047
|
+
const retainedRecovery = activeResource ? recoveryCauses[activeResource] : null;
|
|
17048
|
+
const activeRecovery = retainedRecovery?.origin === "source" && retainedRecovery.tab !== tab.key
|
|
17049
|
+
? null : retainedRecovery;
|
|
15563
17050
|
const activeRequestStatus = requestStatuses[tab.key];
|
|
15564
17051
|
// Once any endpoint proves the folder has no remote, the whole dashboard is
|
|
15565
17052
|
// in setup mode. Keeping this tab-local made a quick tab switch replace the
|
|
@@ -15590,7 +17077,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15590
17077
|
// Counts the lines actually rendered, not the notes collected -- getting this
|
|
15591
17078
|
// wrong by one row is what makes ink repaint the whole frame.
|
|
15592
17079
|
const extraLines =
|
|
15593
|
-
|
|
17080
|
+
(activeRecovery || tabError || activeGovernorDecision?.coordinationError ||
|
|
17081
|
+
activeGovernorDecision?.mode === "paused" || activeGovernorDecision?.disconnected ||
|
|
17082
|
+
data[tab.key] !== null && !runtimeIdentityCoordinator?.current()
|
|
17083
|
+
? 3 : NOTICE_ROWS) +
|
|
15594
17084
|
(tab.key === "security" && !remoteSetup ? securityLines.length : 0);
|
|
15595
17085
|
// Reserve lines for: the tab bar and the divider under it (2), the panel's
|
|
15596
17086
|
// top and bottom edges (2), the column header and its separator (2), and
|
|
@@ -15641,6 +17131,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
15641
17131
|
const ageMs = collectorCachedSourceAge(cachedTab);
|
|
15642
17132
|
return ageMs === null ? [] : [[candidate.key, { ageMs, measuredAt: performance.now() }]];
|
|
15643
17133
|
})));
|
|
17134
|
+
// Keep server source timestamps separate from client-local age estimates.
|
|
17135
|
+
// An SSH receipt's age is measured on the client and cannot prove a new
|
|
17136
|
+
// source observation when a cached snapshot is delivered again.
|
|
17137
|
+
const collectorSourceSuccessRef = useRef(Object.fromEntries(TABS.flatMap((candidate) => {
|
|
17138
|
+
const cachedTab = cachedEntry?.tabs[candidate.key];
|
|
17139
|
+
const sourceAt = cachedTab?.source?.lastSuccessAt ?? cachedTab?.lastOk;
|
|
17140
|
+
return Number.isFinite(sourceAt) ? [[candidate.key, sourceAt]] : [];
|
|
17141
|
+
})));
|
|
15644
17142
|
const fetchTabRef = useRef(null);
|
|
15645
17143
|
const contextCoordinatorRef = useRef(null);
|
|
15646
17144
|
const governorRef = useRef(null);
|
|
@@ -16106,6 +17604,53 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16106
17604
|
: { ...current, [key]: value });
|
|
16107
17605
|
}
|
|
16108
17606
|
|
|
17607
|
+
const scheduledControlAttempts = new Map();
|
|
17608
|
+
|
|
17609
|
+
function recordRuntimeRecovery(key, reason, { retryAt = null, origin = "coordination" } = {}) {
|
|
17610
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
|
|
17611
|
+
if (!resource) return;
|
|
17612
|
+
const at = Date.now();
|
|
17613
|
+
const previous = recoveryRef.current[resource];
|
|
17614
|
+
const candidate = recoveryCause({
|
|
17615
|
+
reason, resource, at, retryAt, sourceAt: lastOkRef.current[key] ?? null,
|
|
17616
|
+
origin, tab: origin === "source" ? key : null,
|
|
17617
|
+
});
|
|
17618
|
+
const hardFailure = ["corrupt", "unwritable", "capacity", "accounting-overflow"].includes(candidate.code);
|
|
17619
|
+
if (!hardFailure && previous?.code === "receipt-retry" &&
|
|
17620
|
+
terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
|
|
17621
|
+
if (!hardFailure && previous?.code === "request-queue" &&
|
|
17622
|
+
cleanupQueueForResource(cleanupQueue, resource) > 0) return;
|
|
17623
|
+
if (previous?.origin === "source" && previous.tab === key && origin === "coordination" &&
|
|
17624
|
+
!["corrupt", "unwritable", "capacity", "accounting-overflow",
|
|
17625
|
+
"receipt-retry", "interrupted-request"].includes(candidate.code)) return;
|
|
17626
|
+
if (previous?.code === candidate.code && previous?.retryAt === candidate.retryAt &&
|
|
17627
|
+
previous?.origin === candidate.origin && previous?.tab === candidate.tab &&
|
|
17628
|
+
at - previous.at < 60_000) return;
|
|
17629
|
+
const cause = recoveryCause({ ...candidate, reason: candidate.code,
|
|
17630
|
+
terminalizationBacklog: cleanupQueueForResource(cleanupQueue, resource) +
|
|
17631
|
+
terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource),
|
|
17632
|
+
...recoveryLedgerSummary(scope, resource, at) });
|
|
17633
|
+
cause.firstAt = previous?.code === cause.code && previous?.origin === cause.origin &&
|
|
17634
|
+
previous?.tab === cause.tab ? previous.firstAt : at;
|
|
17635
|
+
recoveryRef.current = { ...recoveryRef.current, [resource]: cause };
|
|
17636
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17637
|
+
if (scope) setTimeout(() => { safeRecordRecoveryTransition(scope, cause); }, 0).unref?.();
|
|
17638
|
+
}
|
|
17639
|
+
|
|
17640
|
+
function clearRuntimeRecovery(resource, { sourceSuccess = false, tab = null, scheduledHold = null } = {}) {
|
|
17641
|
+
const previous = recoveryRef.current[resource];
|
|
17642
|
+
if (previous?.code === "receipt-retry" &&
|
|
17643
|
+
terminalizationBacklogForResource(terminalizationBacklog, scope?.hash, resource) > 0) return;
|
|
17644
|
+
if (previous?.code === "request-queue" &&
|
|
17645
|
+
cleanupQueueForResource(cleanupQueue, resource) > 0) return;
|
|
17646
|
+
if (!shouldClearRecoveryCause(previous, { sourceSuccess, tab, scheduledHold })) return;
|
|
17647
|
+
recoveryRef.current = { ...recoveryRef.current, [resource]: null };
|
|
17648
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17649
|
+
if (scope) setTimeout(() => {
|
|
17650
|
+
safeRecordRecoveryTransition(scope, { resource, clear: true, at: Date.now() });
|
|
17651
|
+
}, 0).unref?.();
|
|
17652
|
+
}
|
|
17653
|
+
|
|
16109
17654
|
function publishGovernorEpochs(snapshot) {
|
|
16110
17655
|
const epochs = snapshot?.value?.epochs;
|
|
16111
17656
|
if (!isRecord(epochs)) return;
|
|
@@ -16135,7 +17680,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16135
17680
|
return null;
|
|
16136
17681
|
}
|
|
16137
17682
|
|
|
16138
|
-
function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false) {
|
|
17683
|
+
function pauseCoordination(key, reason, diagnosticHold = null, acquisitionFailure = false, retryAt = null) {
|
|
17684
|
+
recordRuntimeRecovery(key, acquisitionFailure && reason === "busy" ? "acquisition-busy" : reason,
|
|
17685
|
+
{ retryAt });
|
|
16139
17686
|
if (acquisitionFailure) setAcquisitionFailures((current) => current[key] === reason
|
|
16140
17687
|
? current : { ...current, [key]: reason ?? "unavailable" });
|
|
16141
17688
|
const hold = diagnosticHold ?? (
|
|
@@ -16180,16 +17727,27 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16180
17727
|
return;
|
|
16181
17728
|
}
|
|
16182
17729
|
if (!snapshot?.ok) {
|
|
16183
|
-
pauseCoordination(key, snapshot?.reason);
|
|
17730
|
+
pauseCoordination(key, snapshot?.reason, null, false, snapshot?.retryAt);
|
|
16184
17731
|
return;
|
|
16185
17732
|
}
|
|
16186
17733
|
const tabReady = governorTabControlReady(snapshot, key, nowMs);
|
|
16187
17734
|
if (!refreshed?.ok && !tabReady) {
|
|
16188
|
-
|
|
17735
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0 &&
|
|
17736
|
+
snapshot.value.observers[candidate]?.outcome !== "healthy");
|
|
17737
|
+
pauseCoordination(key, refreshed?.reason, "observer", false,
|
|
17738
|
+
resource ? snapshot.value.observers[resource]?.nextAt : null);
|
|
16189
17739
|
return;
|
|
16190
17740
|
}
|
|
16191
17741
|
publishGovernorEpochs(snapshot);
|
|
16192
17742
|
if (refreshed.value?.status === "waiting" && !tabReady) {
|
|
17743
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
|
|
17744
|
+
if (resource && !recoveryRef.current[resource] && scope) {
|
|
17745
|
+
const shared = readRecoveryTransitions(scope).current[resource];
|
|
17746
|
+
if (shared?.origin === "coordination") {
|
|
17747
|
+
recoveryRef.current = { ...recoveryRef.current, [resource]: shared };
|
|
17748
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17749
|
+
}
|
|
17750
|
+
}
|
|
16193
17751
|
setTabGovernorDecision(key, { mode: "waiting", probing: true });
|
|
16194
17752
|
return;
|
|
16195
17753
|
}
|
|
@@ -16198,32 +17756,45 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16198
17756
|
const reservation = pendingIntent
|
|
16199
17757
|
? snapshot.value.reservations[`reservation:${pendingIntent.intentId}`]
|
|
16200
17758
|
: null;
|
|
17759
|
+
const costs = tabRequestCost(key);
|
|
16201
17760
|
if (reservation?.status === "scheduled" && reservation.notBefore > nowMs) {
|
|
17761
|
+
const accessKey = pendingIntent.cleanupScope?.accessKey;
|
|
17762
|
+
const reservationId = `reservation:${pendingIntent.intentId}`;
|
|
17763
|
+
const lastAttempt = scheduledControlAttempts.get(key);
|
|
17764
|
+
if (accessKey && pendingIntent.cleanupScope?.hash === scope?.hash &&
|
|
17765
|
+
identity()?.accessKey === accessKey &&
|
|
17766
|
+
(lastAttempt?.id !== reservationId || nowMs - lastAttempt.at >= 30_000)) {
|
|
17767
|
+
scheduledControlAttempts.set(key, { id: reservationId, at: nowMs });
|
|
17768
|
+
// A guarded hold write can retire an older transient notice. Merely
|
|
17769
|
+
// reading a future grant or an absent hold cannot prove recovery.
|
|
17770
|
+
const cleared = clearScheduledRuntimeControlHold(acquisitionEngine,
|
|
17771
|
+
acquisitionSubscriptions, key, accessKey, nowMs);
|
|
17772
|
+
for (const resource of RATE_RESOURCES) {
|
|
17773
|
+
if (costs[resource] > 0) clearRuntimeRecovery(resource, { scheduledHold: cleared });
|
|
17774
|
+
}
|
|
17775
|
+
}
|
|
16202
17776
|
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
16203
17777
|
status: "scheduled",
|
|
16204
|
-
reservationId
|
|
17778
|
+
reservationId,
|
|
16205
17779
|
...reservation,
|
|
16206
17780
|
...currentSharedLaneProvenance(pendingIntent, snapshot.value.leases, nowMs),
|
|
16207
17781
|
}));
|
|
16208
17782
|
return;
|
|
16209
17783
|
}
|
|
16210
|
-
const costs = tabRequestCost(key);
|
|
16211
17784
|
for (const resource of RATE_RESOURCES) {
|
|
16212
17785
|
if (costs[resource] <= 0) continue;
|
|
16213
17786
|
const budget = snapshot.value.budgets[resource];
|
|
16214
17787
|
const decision = budget
|
|
16215
|
-
?
|
|
16216
|
-
budget,
|
|
16217
|
-
resource,
|
|
16218
|
-
leases: snapshot.value.leases,
|
|
16219
|
-
reservations: Object.values(snapshot.value.reservations),
|
|
16220
|
-
nowMs,
|
|
16221
|
-
cost: costs[resource],
|
|
16222
|
-
})
|
|
17788
|
+
? governorResourceDecision(snapshot.value, resource, nowMs, costs[resource])
|
|
16223
17789
|
: { mode: "probe", reason: "budget-unknown" };
|
|
16224
17790
|
if (decision.mode === "open") continue;
|
|
16225
17791
|
const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision.reason) ||
|
|
16226
17792
|
budget?.blockUntil > nowMs;
|
|
17793
|
+
const holdReason = budget?.blockUntil > nowMs
|
|
17794
|
+
? budget.blockReason ?? decision.reason : decision.reason;
|
|
17795
|
+
recordRuntimeRecovery(key, holdReason, {
|
|
17796
|
+
retryAt: recoveryRetryAt(decision, budget, snapshot.value.observers[resource], nowMs),
|
|
17797
|
+
});
|
|
16227
17798
|
setRuntimeAcquisitionHold(
|
|
16228
17799
|
acquisitionEngine,
|
|
16229
17800
|
acquisitionSubscriptions,
|
|
@@ -16235,7 +17806,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16235
17806
|
);
|
|
16236
17807
|
setTabGovernorDecision(key, {
|
|
16237
17808
|
mode: resetHold || decision.mode === "paused" ? "paused" : "waiting",
|
|
16238
|
-
reason:
|
|
17809
|
+
reason: holdReason,
|
|
16239
17810
|
resetMs: budget?.resetMs ?? decision.resetMs ?? null,
|
|
16240
17811
|
probing: decision.mode === "probe" && !resetHold,
|
|
16241
17812
|
});
|
|
@@ -16249,6 +17820,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16249
17820
|
identity()?.accessKey ?? null,
|
|
16250
17821
|
);
|
|
16251
17822
|
setTabGovernorDecision(key, null);
|
|
17823
|
+
for (const resource of RATE_RESOURCES) {
|
|
17824
|
+
if (costs[resource] > 0) clearRuntimeRecovery(resource);
|
|
17825
|
+
}
|
|
16252
17826
|
}
|
|
16253
17827
|
|
|
16254
17828
|
async function refreshRuntimeBudget(currentScope) {
|
|
@@ -16260,15 +17834,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16260
17834
|
return refreshed;
|
|
16261
17835
|
}
|
|
16262
17836
|
|
|
16263
|
-
function
|
|
17837
|
+
function syncRuntimeQuotaProjection(currentScope, snapshot) {
|
|
16264
17838
|
if (!snapshot?.ok) return;
|
|
16265
|
-
|
|
16266
|
-
const observer = snapshot.value.observers?.[resource];
|
|
16267
|
-
const epoch = snapshot.value.epochs?.[resource];
|
|
16268
|
-
return observer?.outcome === "healthy" && typeof epoch === "string" && Number.isFinite(observer.at)
|
|
16269
|
-
? [[resource, { epoch, observedAt: observer.at }]] : [];
|
|
16270
|
-
}));
|
|
16271
|
-
acquisitionEngine.reconcileUncertainty(currentScope.accessKey, evidence);
|
|
17839
|
+
acquisitionEngine.syncQuotaProjection(currentScope);
|
|
16272
17840
|
}
|
|
16273
17841
|
|
|
16274
17842
|
function commit(key, run, {
|
|
@@ -16320,7 +17888,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16320
17888
|
: run;
|
|
16321
17889
|
return requestIdentityStorage.run(scope, fencedRun)
|
|
16322
17890
|
.then(async (result) => {
|
|
16323
|
-
|
|
17891
|
+
const terminal = queueTerminalization(settlementScope, scope.receiptCapability, result?.measuredSuccess === false
|
|
16324
17892
|
? { outcome: "rejected", observations: result?.observations ?? [] }
|
|
16325
17893
|
: {
|
|
16326
17894
|
outcome: "measured-success",
|
|
@@ -16329,7 +17897,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16329
17897
|
graphql: result?.graphqlSpent ?? GRAPHQL_PER_FETCH[key] ?? 0,
|
|
16330
17898
|
},
|
|
16331
17899
|
observations: result?.observations ?? [],
|
|
16332
|
-
}
|
|
17900
|
+
});
|
|
17901
|
+
if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
|
|
17902
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
17903
|
+
failAcquisition(acquisition, { requestMetrics: acquisitionRequestMetrics(result), hold: "primary" });
|
|
17904
|
+
pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
|
|
17905
|
+
return;
|
|
17906
|
+
}
|
|
16333
17907
|
if (!currentAccess()) {
|
|
16334
17908
|
failAcquisition(acquisition, {
|
|
16335
17909
|
requestMetrics: acquisitionRequestMetrics(result),
|
|
@@ -16352,7 +17926,6 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16352
17926
|
acquisition.capabilities = result.capabilities ?? {};
|
|
16353
17927
|
acquisition.repositoryIdentity = result.repositoryIdentity;
|
|
16354
17928
|
acquisition.requestMetrics = acquisitionRequestMetrics(result);
|
|
16355
|
-
acquisition.uncertainReceipts = result.uncertainReceipts ?? [];
|
|
16356
17929
|
}
|
|
16357
17930
|
// Identical payload: skip the parse *and* the state update. Returning
|
|
16358
17931
|
// the same state object makes React bail out of the re-render, so an
|
|
@@ -16387,9 +17960,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16387
17960
|
// so the next admitted tick parses its response instead of taking the
|
|
16388
17961
|
// identical-output fast path.
|
|
16389
17962
|
if (transition.kind === "unusable") {
|
|
17963
|
+
recordRuntimeRecovery(key, "source-incomplete", { origin: "source" });
|
|
16390
17964
|
failAcquisition(acquisition, {
|
|
16391
17965
|
requestMetrics: acquisition?.requestMetrics ?? {},
|
|
16392
|
-
uncertainReceipts: acquisition?.uncertainReceipts ?? [],
|
|
16393
17966
|
hold: "primary",
|
|
16394
17967
|
});
|
|
16395
17968
|
rawRef.current[key] = transition.nextRaw;
|
|
@@ -16402,9 +17975,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16402
17975
|
// freshness. Do not retain its raw value either, so the next source
|
|
16403
17976
|
// retry is parsed instead of taking the identical-payload fast path.
|
|
16404
17977
|
if (transition.kind === "blind") {
|
|
17978
|
+
recordRuntimeRecovery(key, "security-incomplete", { origin: "source" });
|
|
16405
17979
|
failAcquisition(acquisition, {
|
|
16406
17980
|
requestMetrics: acquisition?.requestMetrics ?? {},
|
|
16407
|
-
uncertainReceipts: acquisition?.uncertainReceipts ?? [],
|
|
16408
17981
|
hold: "primary",
|
|
16409
17982
|
});
|
|
16410
17983
|
rawRef.current[key] = transition.nextRaw;
|
|
@@ -16439,6 +18012,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16439
18012
|
publishStagedEntities(entityRef.current, stagedEntities, transition.kind);
|
|
16440
18013
|
if (transition.kind === "unchanged") {
|
|
16441
18014
|
lastOkRef.current[key] = completedAt;
|
|
18015
|
+
for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
|
|
18016
|
+
clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
|
|
18017
|
+
}
|
|
16442
18018
|
clearBackoff(`tab:${key}`);
|
|
16443
18019
|
setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
|
|
16444
18020
|
const cachedTab = pick(
|
|
@@ -16455,6 +18031,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16455
18031
|
const tabData = transition.data;
|
|
16456
18032
|
const tabMeta = transition.meta;
|
|
16457
18033
|
lastOkRef.current[key] = completedAt;
|
|
18034
|
+
for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
|
|
18035
|
+
clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
|
|
18036
|
+
}
|
|
16458
18037
|
clearBackoff(`tab:${key}`);
|
|
16459
18038
|
rawRef.current[key] = transition.nextRaw;
|
|
16460
18039
|
setErrors((x) => (x[key] === null ? x : { ...x, [key]: null }));
|
|
@@ -16474,10 +18053,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16474
18053
|
requestMetrics: err?.requestMetrics ?? {},
|
|
16475
18054
|
hold: acquisitionFailureHold(err),
|
|
16476
18055
|
});
|
|
16477
|
-
|
|
18056
|
+
const terminal = queueTerminalization(settlementScope, scope.receiptCapability, {
|
|
16478
18057
|
outcome: governorOutcomeForError(err),
|
|
16479
18058
|
observations: err?.budgetObservations ?? [],
|
|
16480
|
-
}
|
|
18059
|
+
});
|
|
18060
|
+
if (terminal.status === "retryable") recordRuntimeRecovery(key, "receipt-retry");
|
|
18061
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
18062
|
+
pauseCoordination(key, terminalizationRecoveryCode(terminal), null, true);
|
|
18063
|
+
}
|
|
16481
18064
|
if (cancelled || !currentAccess() || err?.name === "AbortError") return;
|
|
16482
18065
|
// Preserve both the verdict and the bounded raw error in state. The
|
|
16483
18066
|
// renderer translates recognized verdicts at draw time, which lets a
|
|
@@ -16494,6 +18077,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16494
18077
|
// recover on the very next tick once the network is back.
|
|
16495
18078
|
const steps = verdict === "rate-limited" ? null : pick(FAILURE_LADDER, verdict, null);
|
|
16496
18079
|
if (steps) recordFailure(`tab:${key}`, performance.now(), steps);
|
|
18080
|
+
const backoffUntil = alertBackoff.get(backoffStorageKey(`tab:${key}`))?.until;
|
|
18081
|
+
const retryAt = Number.isFinite(backoffUntil)
|
|
18082
|
+
? Date.now() + Math.max(0, backoffUntil - performance.now()) : null;
|
|
18083
|
+
if (terminal.status !== "blocked" && terminal.status !== "compacted") {
|
|
18084
|
+
const recovery = requestFailureRecovery(err, verdict);
|
|
18085
|
+
recordRuntimeRecovery(key, recovery.reason, { origin: recovery.origin, retryAt });
|
|
18086
|
+
}
|
|
16497
18087
|
if (verdict === "rate-limited") {
|
|
16498
18088
|
const snapshot = inspectGovernor(scope, Date.now());
|
|
16499
18089
|
const costs = tabRequestCost(key);
|
|
@@ -16508,6 +18098,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16508
18098
|
});
|
|
16509
18099
|
}
|
|
16510
18100
|
handleRateLimitBlocks(key, blocks, Date.now());
|
|
18101
|
+
const held = inspectGovernor(scope, Date.now());
|
|
18102
|
+
const heldUntil = RATE_RESOURCES.filter((resource) => costs[resource] > 0)
|
|
18103
|
+
.map((resource) => held.value?.budgets?.[resource]?.blockUntil)
|
|
18104
|
+
.filter((deadline) => Number.isFinite(deadline) && deadline > Date.now());
|
|
18105
|
+
if (heldUntil.length > 0) recordRuntimeRecovery(key, "rate-limited", {
|
|
18106
|
+
origin: "source", retryAt: Math.min(...heldUntil),
|
|
18107
|
+
});
|
|
16511
18108
|
}
|
|
16512
18109
|
})
|
|
16513
18110
|
.finally(() => {
|
|
@@ -16541,6 +18138,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16541
18138
|
const leaseId = governorId();
|
|
16542
18139
|
const pending = new Map();
|
|
16543
18140
|
const cleanupQueue = new Set();
|
|
18141
|
+
let lastWallAt = Date.now();
|
|
16544
18142
|
// One manual handoff per tab, carrying which key produced it. A keypress
|
|
16545
18143
|
// during an automatic request must not be lost, but repeated keypresses
|
|
16546
18144
|
// during the manual request itself must not create a trailing second batch.
|
|
@@ -16579,12 +18177,20 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16579
18177
|
const unchangedPolls = Object.fromEntries(TAB_KEYS.map((key) => [key, 0]));
|
|
16580
18178
|
let pendingBlockPublications = new Map();
|
|
16581
18179
|
const acquisitionSubscriptions = new Map();
|
|
18180
|
+
// One-shot demand survives admission delays, but ends when an observation arrives.
|
|
18181
|
+
const requestedTabs = new Map();
|
|
16582
18182
|
|
|
16583
18183
|
function adoptAcquisitionSnapshot(key, snapshot, receipt = null) {
|
|
16584
18184
|
if (!snapshot || cancelled) return;
|
|
18185
|
+
if (requestedTabs.has(key) && !queuedManual.has(key) &&
|
|
18186
|
+
snapshot.lastSuccessAt >= requestedTabs.get(key)) {
|
|
18187
|
+
requestedTabs.delete(key);
|
|
18188
|
+
if (runtime.background === "on-demand" && key !== "actions") pollDueAt[key] = Infinity;
|
|
18189
|
+
}
|
|
16585
18190
|
if (runtime.connect === null && Number.isFinite(snapshot.nextDueAt)) {
|
|
16586
18191
|
sharedSnapshotDueAt = { ...sharedSnapshotDueAt, [key]: snapshot.nextDueAt };
|
|
16587
|
-
if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key
|
|
18192
|
+
if (pollDeadlinesOpen && key === TABS[activeIndexRef.current].key &&
|
|
18193
|
+
Number.isFinite(pollIntervalFor(key, key))) {
|
|
16588
18194
|
const at = Date.now();
|
|
16589
18195
|
const due = sharedAcquisitionPollDeadline(pollDueAt[key], snapshot.nextDueAt, at, true);
|
|
16590
18196
|
if (due < pollDueAt[key]) {
|
|
@@ -16657,6 +18263,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16657
18263
|
const host = effectiveRuntimeHost({ remoteUrls: runtimeRemoteUrls });
|
|
16658
18264
|
const handles = new Map();
|
|
16659
18265
|
const receivedSnapshots = new Set();
|
|
18266
|
+
const initialKey = TABS[activeIndexRef.current].key;
|
|
18267
|
+
const collectorDemand = (key) => ({
|
|
18268
|
+
active: TABS[activeIndexRef.current].key === key &&
|
|
18269
|
+
(runtime.background !== "on-demand" || key === "actions"),
|
|
18270
|
+
background: runtime.background === "all",
|
|
18271
|
+
floorMs: runtime.refreshMs,
|
|
18272
|
+
pages: pageStateRef.current[key]?.pages ?? 1,
|
|
18273
|
+
});
|
|
16660
18274
|
let collectorReady = false;
|
|
16661
18275
|
setGovernorDecisions(Object.fromEntries(TAB_KEYS.map((key) => [key, { disconnected: true }])));
|
|
16662
18276
|
if (!repository || !host) {
|
|
@@ -16686,39 +18300,46 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16686
18300
|
},
|
|
16687
18301
|
});
|
|
16688
18302
|
for (const key of TAB_KEYS) {
|
|
16689
|
-
const demand =
|
|
16690
|
-
active: TABS[activeIndexRef.current].key === key,
|
|
16691
|
-
background: runtime.background !== "off",
|
|
16692
|
-
floorMs: runtime.refreshMs,
|
|
16693
|
-
pages: pageStateRef.current[key]?.pages ?? 1,
|
|
16694
|
-
};
|
|
18303
|
+
const demand = collectorDemand(key);
|
|
16695
18304
|
const subscribed = client.subscribe({ host, repo: repository, resource: key, demand,
|
|
16696
18305
|
sourceCheckpoint: cachedEntry?.tabs[key]?.source ?? null,
|
|
16697
18306
|
onSnapshot: (snapshot, receipt) => {
|
|
16698
18307
|
receivedSnapshots.add(key);
|
|
18308
|
+
const sourceAdvanced = collectorSourceAdvanced(
|
|
18309
|
+
collectorSourceSuccessRef.current[key], snapshot.lastSuccessAt);
|
|
16699
18310
|
adoptAcquisitionSnapshot(key, snapshot, receipt);
|
|
18311
|
+
if (Number.isFinite(snapshot.lastSuccessAt) &&
|
|
18312
|
+
(!Number.isFinite(collectorSourceSuccessRef.current[key]) ||
|
|
18313
|
+
snapshot.lastSuccessAt > collectorSourceSuccessRef.current[key])) {
|
|
18314
|
+
collectorSourceSuccessRef.current[key] = snapshot.lastSuccessAt;
|
|
18315
|
+
}
|
|
18316
|
+
if (sourceAdvanced) {
|
|
18317
|
+
for (const resource of RATE_RESOURCES) if (tabRequestCost(key)[resource] > 0) {
|
|
18318
|
+
clearRuntimeRecovery(resource, { sourceSuccess: true, tab: key });
|
|
18319
|
+
}
|
|
18320
|
+
}
|
|
16700
18321
|
setErrors((current) => current[key] === null ? current : { ...current, [key]: null });
|
|
16701
18322
|
setWaiting((current) => current[key] ? { ...current, [key]: false } : current);
|
|
16702
18323
|
setGovernorDecisions((current) => ({ ...current, [key]:
|
|
16703
18324
|
collectorDisplayDecision({ connected: collectorReady, hasSnapshot: true }) }));
|
|
16704
18325
|
},
|
|
16705
|
-
onHold: (hold) =>
|
|
16706
|
-
|
|
16707
|
-
|
|
18326
|
+
onHold: (hold) => {
|
|
18327
|
+
if (hold === "app-auth") recordRuntimeRecovery(key, "app-auth", { origin: "source" });
|
|
18328
|
+
setGovernorDecisions((current) => ({ ...current, [key]:
|
|
18329
|
+
collectorDisplayDecision({ connected: collectorReady,
|
|
18330
|
+
hasSnapshot: receivedSnapshots.has(key), hold }) }));
|
|
18331
|
+
},
|
|
16708
18332
|
});
|
|
16709
18333
|
if (subscribed.ok) handles.set(key, subscribed.value);
|
|
16710
18334
|
}
|
|
18335
|
+
if (runtime.background === "on-demand" && initialKey !== "actions") {
|
|
18336
|
+
handles.get(initialKey)?.refresh();
|
|
18337
|
+
}
|
|
16711
18338
|
fetchTabRef.current = (key, request = {}) => {
|
|
16712
18339
|
const handle = handles.get(key);
|
|
16713
18340
|
if (!handle) return;
|
|
16714
|
-
|
|
16715
|
-
|
|
16716
|
-
candidateHandle.updateDemand({ active: candidate === key, background: runtime.background !== "off",
|
|
16717
|
-
floorMs: runtime.refreshMs, pages: pageStateRef.current[candidate]?.pages ?? 1 });
|
|
16718
|
-
}
|
|
16719
|
-
} else {
|
|
16720
|
-
handle.updateDemand({ active: true, background: runtime.background !== "off",
|
|
16721
|
-
floorMs: runtime.refreshMs, pages: pageStateRef.current[key]?.pages ?? 1 });
|
|
18341
|
+
for (const [candidate, candidateHandle] of handles) {
|
|
18342
|
+
candidateHandle.updateDemand(collectorDemand(candidate));
|
|
16722
18343
|
}
|
|
16723
18344
|
handle.refresh(request.force === true);
|
|
16724
18345
|
};
|
|
@@ -16795,12 +18416,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16795
18416
|
demand: acquisition.demand?.active ? "active" : "inactive",
|
|
16796
18417
|
unchangedCount: publicationView.unchangedCount,
|
|
16797
18418
|
inProgressCI: key === "actions" && actionsInProgress(rows),
|
|
16798
|
-
background: runtime.background,
|
|
18419
|
+
background: runtime.background === "on-demand" ? "all" : runtime.background,
|
|
16799
18420
|
}),
|
|
16800
18421
|
hold: null,
|
|
16801
18422
|
capabilities: acquisition.capabilities ?? {},
|
|
16802
18423
|
requestMetrics: acquisition.requestMetrics ?? {},
|
|
16803
|
-
uncertainReceipts: acquisition.uncertainReceipts ?? [],
|
|
16804
18424
|
repositoryIdentity: acquisition.repositoryIdentity,
|
|
16805
18425
|
meta,
|
|
16806
18426
|
securityNotes: key === "security"
|
|
@@ -16841,10 +18461,17 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16841
18461
|
automaticStatusVisible) {
|
|
16842
18462
|
const descriptor = tabForKey(key);
|
|
16843
18463
|
let acquisitionStartAttempted = false;
|
|
18464
|
+
let transportScope = null;
|
|
16844
18465
|
const transported = await runStartedAcquisitionTransport(
|
|
16845
18466
|
() => {
|
|
16846
18467
|
const admitted = inspectGovernor(currentScope, nowMs);
|
|
16847
18468
|
if (!admitted.ok) return admitted;
|
|
18469
|
+
const admittedReservation = admitted.value.reservations[reservationId];
|
|
18470
|
+
if (!admittedReservation?.receipt || admittedReservation.status !== "started") {
|
|
18471
|
+
return { ok: false, reason: "compacted" };
|
|
18472
|
+
}
|
|
18473
|
+
transportScope = { ...currentScope,
|
|
18474
|
+
receiptCapability: receiptCapabilityFromReservation(reservationId, admittedReservation) };
|
|
16848
18475
|
acquisitionStartAttempted = true;
|
|
16849
18476
|
const receipt = {
|
|
16850
18477
|
reservationId,
|
|
@@ -16875,13 +18502,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16875
18502
|
force: item.force,
|
|
16876
18503
|
catalog: workflowCatalogRef.current,
|
|
16877
18504
|
pages: item.acquisition.demand?.pages ?? pageStateRef.current[key]?.pages ?? 1,
|
|
16878
|
-
governor: { scope:
|
|
18505
|
+
governor: { scope: transportScope, leaseId },
|
|
16879
18506
|
previousRaw: rawRef.current[key] ?? null,
|
|
16880
18507
|
}),
|
|
16881
18508
|
{
|
|
16882
18509
|
force: item.force,
|
|
16883
18510
|
manual: item.manual === true,
|
|
16884
|
-
scope:
|
|
18511
|
+
scope: transportScope,
|
|
16885
18512
|
reservationId,
|
|
16886
18513
|
admittedAt: nowMs,
|
|
16887
18514
|
automaticStatusVisible,
|
|
@@ -16892,6 +18519,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16892
18519
|
},
|
|
16893
18520
|
);
|
|
16894
18521
|
if (transported.ok) return transported;
|
|
18522
|
+
if (transportScope?.receiptCapability) {
|
|
18523
|
+
const terminal = queueTerminalization(currentScope,
|
|
18524
|
+
transportScope.receiptCapability, { outcome: "rejected", neverStarted: true });
|
|
18525
|
+
if (terminal.status === "blocked" || terminal.status === "compacted") {
|
|
18526
|
+
pauseCoordination(key, terminal.reason ?? terminal.status, null, true);
|
|
18527
|
+
}
|
|
18528
|
+
}
|
|
16895
18529
|
pauseCoordination(key, transported.reason, null, acquisitionStartAttempted);
|
|
16896
18530
|
// The governor reservation is already started. Even a transient failed
|
|
16897
18531
|
// claim write cannot safely rebind it; retain its charge and get a fresh
|
|
@@ -16915,6 +18549,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16915
18549
|
demand: key === activeKey ? "active" : "inactive",
|
|
16916
18550
|
unchangedCount: unchangedPolls[key],
|
|
16917
18551
|
inProgressCI: key === "actions" && actionsInProgress(actionRows),
|
|
18552
|
+
requested: requestedTabs.has(key),
|
|
16918
18553
|
background: runtime.background,
|
|
16919
18554
|
});
|
|
16920
18555
|
}
|
|
@@ -16925,7 +18560,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16925
18560
|
const localDueAt = Number.isFinite(step) ? at + step : Number.POSITIVE_INFINITY;
|
|
16926
18561
|
pollDueAt = {
|
|
16927
18562
|
...pollDueAt,
|
|
16928
|
-
[key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active),
|
|
18563
|
+
[key]: sharedAcquisitionPollDeadline(localDueAt, sharedSnapshotDueAt[key], at, active && Number.isFinite(step)),
|
|
16929
18564
|
};
|
|
16930
18565
|
}
|
|
16931
18566
|
|
|
@@ -16937,6 +18572,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
16937
18572
|
function openPollDeadlines(at) {
|
|
16938
18573
|
const activeKey = TABS[activeIndexRef.current].key;
|
|
16939
18574
|
const opened = { ...NEVER_DUE };
|
|
18575
|
+
if (runtime.background === "on-demand") requestedTabs.set(activeKey, at);
|
|
16940
18576
|
opened[activeKey] = at + 1;
|
|
16941
18577
|
let slot = 1;
|
|
16942
18578
|
for (const key of TAB_KEYS) {
|
|
@@ -17038,6 +18674,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17038
18674
|
coordinator.invalidate();
|
|
17039
18675
|
setFailureContext(null);
|
|
17040
18676
|
if (discardData) {
|
|
18677
|
+
recoveryRef.current = { core: null, graphql: null };
|
|
18678
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17041
18679
|
rawRef.current = {};
|
|
17042
18680
|
entityRef.current.clear();
|
|
17043
18681
|
lastOkRef.current = {};
|
|
@@ -17115,6 +18753,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17115
18753
|
}
|
|
17116
18754
|
scope = createQuotaScope(current, { root: runtimeIdentityCoordinator.root, identityProvider: identity });
|
|
17117
18755
|
cleanupScope = { ...scope, identityProvider: null };
|
|
18756
|
+
const savedRecovery = readRecoveryTransitions(scope);
|
|
18757
|
+
recoveryRef.current = savedRecovery.status === "healthy"
|
|
18758
|
+
? { ...savedRecovery.current } : { core: null, graphql: null };
|
|
18759
|
+
setRecoveryCauses(recoveryRef.current);
|
|
17118
18760
|
const nextTarget = dashboardCacheTarget({ repo: runtime.repo, ghRepo: process.env.GH_REPO, host: current.host, account: current.accessKey });
|
|
17119
18761
|
// The mount-time target had to guess the account, and guesses
|
|
17120
18762
|
// "unverified" whenever the bounded warm resolution above did not land
|
|
@@ -17176,6 +18818,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17176
18818
|
function finishPendingCleanup(key, item) {
|
|
17177
18819
|
if (pending.get(key) === item) pending.delete(key);
|
|
17178
18820
|
cleanupQueue.delete(item);
|
|
18821
|
+
const resource = RATE_RESOURCES.find((candidate) => tabRequestCost(key)[candidate] > 0);
|
|
18822
|
+
if (resource && cleanupQueueForResource(cleanupQueue, resource) === 0 &&
|
|
18823
|
+
recoveryRef.current[resource]?.code === "request-queue") {
|
|
18824
|
+
clearRuntimeRecovery(resource);
|
|
18825
|
+
}
|
|
17179
18826
|
setAcquisitionFailures((current) => {
|
|
17180
18827
|
if (!Object.hasOwn(current, key)) return current;
|
|
17181
18828
|
const next = { ...current };
|
|
@@ -17204,7 +18851,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17204
18851
|
}
|
|
17205
18852
|
|
|
17206
18853
|
function retryPendingCleanup(key, item, at = Date.now()) {
|
|
17207
|
-
if (item.cleanupInFlight)
|
|
18854
|
+
if (item.cleanupInFlight) {
|
|
18855
|
+
recordRuntimeRecovery(key, "request-queue");
|
|
18856
|
+
return false;
|
|
18857
|
+
}
|
|
17208
18858
|
item.cleanupPending = true;
|
|
17209
18859
|
item.cleanupKey = key;
|
|
17210
18860
|
cleanupQueue.add(item);
|
|
@@ -17230,6 +18880,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17230
18880
|
finishPendingCleanup(key, item);
|
|
17231
18881
|
}
|
|
17232
18882
|
if (!item.governorClean) armWake("data", governorControlRetryAt(at, runtime.refreshMs), dataWake);
|
|
18883
|
+
if (pending.has(key)) recordRuntimeRecovery(key, "request-queue");
|
|
17233
18884
|
return !pending.has(key);
|
|
17234
18885
|
}
|
|
17235
18886
|
|
|
@@ -17252,7 +18903,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17252
18903
|
// the tab's failure ladder. `force` is the separate, stronger request that
|
|
17253
18904
|
// also drops validators -- the `R` key. `r` is manual and not forced, which
|
|
17254
18905
|
// is what makes a quiet refresh cost nothing.
|
|
17255
|
-
async function requestTab(key, kind = "active", { force = false } = {}) {
|
|
18906
|
+
async function requestTab(key, kind = "active", { force = false, detachTransport = false } = {}) {
|
|
17256
18907
|
const manual = kind === "manual";
|
|
17257
18908
|
const signal = controller.signal;
|
|
17258
18909
|
const monotonicNow = performance.now();
|
|
@@ -17333,7 +18984,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17333
18984
|
currentScope.accessKey,
|
|
17334
18985
|
);
|
|
17335
18986
|
if (ownership.value.role !== "producer") {
|
|
17336
|
-
if (ownership.value.snapshot)
|
|
18987
|
+
if (ownership.value.snapshot) {
|
|
18988
|
+
if (ownership.value.reason === "fresh") requestedTabs.delete(key);
|
|
18989
|
+
adoptAcquisitionSnapshot(key, ownership.value.snapshot);
|
|
18990
|
+
}
|
|
17337
18991
|
if (ownership.value.sharingCount > 1) {
|
|
17338
18992
|
setTabGovernorDecision(key, {
|
|
17339
18993
|
mode: "waiting",
|
|
@@ -17457,9 +19111,19 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17457
19111
|
setTabWaiting(key, false);
|
|
17458
19112
|
const item = pending.get(key);
|
|
17459
19113
|
if (!item) return { persisted: false, retry: false, kind, key };
|
|
17460
|
-
|
|
19114
|
+
const transport = startPendingAcquisitionTransport(
|
|
17461
19115
|
key, item, currentScope, decision.reservationId, nowMs, signal, false,
|
|
17462
19116
|
);
|
|
19117
|
+
if (detachTransport) {
|
|
19118
|
+
// The scheduling single-flight owns admission, not the HTTP lifetime.
|
|
19119
|
+
// A background response must not block the next active-tab wake.
|
|
19120
|
+
void transport.catch((error) => {
|
|
19121
|
+
if (cancelled || pending.get(key) !== item) return;
|
|
19122
|
+
item.retryOnCleanup = true;
|
|
19123
|
+
cancelPending(key, currentScope);
|
|
19124
|
+
pauseCoordination(key, error?.reason ?? "stale");
|
|
19125
|
+
});
|
|
19126
|
+
} else await transport;
|
|
17463
19127
|
return { persisted: true, retry: false, kind, key };
|
|
17464
19128
|
}
|
|
17465
19129
|
fetchTabRef.current = (key, { force = false, kind = force ? "manual" : "active" } = {}) => {
|
|
@@ -17470,6 +19134,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17470
19134
|
unchangedPolls[key] = advanceUnchangedCount(unchangedPolls[key], "subscribed");
|
|
17471
19135
|
}
|
|
17472
19136
|
const requestedAt = Date.now();
|
|
19137
|
+
if (runtime.background === "on-demand") {
|
|
19138
|
+
if (kind === "tab-switch") requestedTabs.clear();
|
|
19139
|
+
requestedTabs.set(key, requestedAt);
|
|
19140
|
+
}
|
|
17473
19141
|
// Manual work replaces the active poll that would otherwise be due. Move
|
|
17474
19142
|
// that deadline before touching the governor so a data wake already in
|
|
17475
19143
|
// this event-loop turn cannot start a second batch as soon as the forced
|
|
@@ -17583,11 +19251,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17583
19251
|
const budget = snapshot.value.budgets[resource];
|
|
17584
19252
|
const observer = snapshot.value.observers[resource];
|
|
17585
19253
|
const decision = budget
|
|
17586
|
-
?
|
|
19254
|
+
? governorAvailableForGrant(snapshot.value, resource, nowMs)
|
|
17587
19255
|
: { mode: "paused" };
|
|
17588
|
-
const held =
|
|
17589
|
-
|
|
17590
|
-
decision.mode === "paused" && decision.reason === "budget-exhausted";
|
|
19256
|
+
const held = governorResourceHeld(snapshot.value, resource, nowMs,
|
|
19257
|
+
pendingBlockPublications.has(resource));
|
|
17591
19258
|
const retryAt = budget?.blockUntil > nowMs
|
|
17592
19259
|
? budget.blockUntil
|
|
17593
19260
|
: observer?.nextAt ?? decision.retryAt ?? wakes.controlAt;
|
|
@@ -17607,13 +19274,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17607
19274
|
states: Object.fromEntries(TAB_KEYS.map((key) => [key, {
|
|
17608
19275
|
unchangedCount: unchangedPolls[key],
|
|
17609
19276
|
inProgressCI: key === "actions" && actionsInProgress(),
|
|
19277
|
+
requested: requestedTabs.has(key),
|
|
17610
19278
|
}])),
|
|
17611
19279
|
});
|
|
17612
19280
|
scheduledWakePolls = planned.due;
|
|
17613
19281
|
pollDueAt = planned.dueAt;
|
|
17614
19282
|
backgroundIndex = planned.backgroundIndex;
|
|
17615
19283
|
const outcomes = await Promise.allSettled(
|
|
17616
|
-
planned.due.map(({ key, kind }) => requestTab(key, kind)),
|
|
19284
|
+
planned.due.map(({ key, kind }) => requestTab(key, kind, { detachTransport: true })),
|
|
17617
19285
|
);
|
|
17618
19286
|
const retryAt = governorControlRetryAt(Date.now(), runtime.refreshMs);
|
|
17619
19287
|
const retried = retryFailedPollBatch({ due: planned.due, outcomes,
|
|
@@ -17666,6 +19334,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17666
19334
|
|
|
17667
19335
|
async function runControlWake() {
|
|
17668
19336
|
if (cancelled) return;
|
|
19337
|
+
const controlAt = Date.now();
|
|
19338
|
+
const clockCause = wallClockRecovery(lastWallAt, controlAt, Math.min(runtime.refreshMs, 5_000));
|
|
19339
|
+
lastWallAt = controlAt;
|
|
19340
|
+
if (clockCause) {
|
|
19341
|
+
recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
|
|
19342
|
+
setNow(new Date(controlAt));
|
|
19343
|
+
}
|
|
17669
19344
|
for (const item of [...cleanupQueue]) retryPendingCleanup(item.cleanupKey, item);
|
|
17670
19345
|
await runtimeIdentityCoordinator.refresh();
|
|
17671
19346
|
if (cancelled) return;
|
|
@@ -17684,11 +19359,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17684
19359
|
}
|
|
17685
19360
|
const refreshed = currentScope
|
|
17686
19361
|
? await refreshRuntimeBudget(currentScope)
|
|
17687
|
-
: { ok: false,
|
|
19362
|
+
: { ok: false, ...(runtimeIdentityCoordinator?.inspect() ?? { reason: "stale" }) };
|
|
17688
19363
|
if (cancelled) return;
|
|
17689
19364
|
const checkedAt = Date.now();
|
|
17690
19365
|
const snapshot = currentScope ? inspectGovernor(currentScope, checkedAt) : refreshed;
|
|
17691
|
-
if (currentScope)
|
|
19366
|
+
if (currentScope) syncRuntimeQuotaProjection(currentScope, snapshot);
|
|
17692
19367
|
if (currentScope && snapshot.ok && pendingBlockPublications.size > 0) {
|
|
17693
19368
|
attemptPendingBlockPublications(currentScope, checkedAt, snapshot.value);
|
|
17694
19369
|
}
|
|
@@ -17750,14 +19425,27 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17750
19425
|
const livenessWake = createRecurringWake({
|
|
17751
19426
|
scheduler: wakeScheduler,
|
|
17752
19427
|
intervalMs: Math.min(runtime.refreshMs, 5000),
|
|
17753
|
-
run:
|
|
19428
|
+
run: () => {
|
|
17754
19429
|
if (cancelled || remoteSetupRef.current) return;
|
|
17755
19430
|
const at = Date.now();
|
|
19431
|
+
const clockCause = wallClockRecovery(lastWallAt, at, Math.min(runtime.refreshMs, 5_000));
|
|
19432
|
+
lastWallAt = at;
|
|
19433
|
+
for (const resource of RATE_RESOURCES) {
|
|
19434
|
+
if (["receipt-retry", "request-queue"].includes(recoveryRef.current[resource]?.code)) {
|
|
19435
|
+
clearRuntimeRecovery(resource);
|
|
19436
|
+
}
|
|
19437
|
+
}
|
|
19438
|
+
if (clockCause) {
|
|
19439
|
+
recordRuntimeRecovery(TABS[activeIndexRef.current].key, clockCause, { origin: "source" });
|
|
19440
|
+
setNow(new Date(at));
|
|
19441
|
+
void controlWake();
|
|
19442
|
+
return;
|
|
19443
|
+
}
|
|
17756
19444
|
if (wakeScheduler.at("control") <= at ||
|
|
17757
|
-
!Number.isFinite(wakeScheduler.at("control")))
|
|
19445
|
+
!Number.isFinite(wakeScheduler.at("control"))) void controlWake();
|
|
17758
19446
|
if (pending.size > 0 || cleanupQueue.size > 0 ||
|
|
17759
19447
|
wakeScheduler.at("data") <= Date.now() ||
|
|
17760
|
-
Object.values(pollDueAt).some((due) => due <= Date.now()))
|
|
19448
|
+
Object.values(pollDueAt).some((due) => due <= Date.now())) void dataWake();
|
|
17761
19449
|
},
|
|
17762
19450
|
onError: (error) => {
|
|
17763
19451
|
if (!cancelled) pauseCoordination(TABS[activeIndexRef.current].key, error?.reason ?? "stale");
|
|
@@ -17773,7 +19461,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17773
19461
|
if (cancelled) return;
|
|
17774
19462
|
const currentScope = ensureScope(Date.now());
|
|
17775
19463
|
if (!currentScope) {
|
|
17776
|
-
|
|
19464
|
+
const identityFailure = runtimeIdentityCoordinator?.inspect();
|
|
19465
|
+
pauseCoordination(TABS[activeIndexRef.current].key,
|
|
19466
|
+
identityFailure?.reason ?? "unknown-scope", null, false, identityFailure?.retryAt);
|
|
17777
19467
|
armWake("control", governorControlRetryAt(Date.now(), runtime.refreshMs), controlWake);
|
|
17778
19468
|
return;
|
|
17779
19469
|
}
|
|
@@ -17783,7 +19473,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17783
19473
|
if (cancelled) return;
|
|
17784
19474
|
const checkedAt = Date.now();
|
|
17785
19475
|
const snapshot = inspectGovernor(currentScope, checkedAt);
|
|
17786
|
-
|
|
19476
|
+
syncRuntimeQuotaProjection(currentScope, snapshot);
|
|
17787
19477
|
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
17788
19478
|
if (snapshot.ok) {
|
|
17789
19479
|
controlEpochs = { ...snapshot.value.epochs };
|
|
@@ -17907,7 +19597,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17907
19597
|
const displayError = formatTabErrorForWidth(tabError, failureContext, Math.max(1, cols - 5));
|
|
17908
19598
|
const coordinationError = activeGovernorDecision?.coordinationError && !remoteSetup;
|
|
17909
19599
|
const coordinationReason = activeFailure ?? activeGovernorDecision?.reason ?? "unavailable";
|
|
17910
|
-
const
|
|
19600
|
+
const noticeReason = activeRecovery?.code ??
|
|
19601
|
+
(tabError?.kind === "fetch" ? tabError.verdict : null) ??
|
|
19602
|
+
(coordinationError || activeGovernorDecision?.mode === "paused" ? coordinationReason : null) ??
|
|
19603
|
+
(statusDisconnected ? runtime.connect === null ? "identity-unavailable"
|
|
19604
|
+
: activeGovernorDecision?.reason === "app-auth" ? "app-auth" : "disconnected" : null);
|
|
19605
|
+
// A continuous fault stays sustained when its explanation becomes more
|
|
19606
|
+
// specific. Only losing the fault (or switching tabs) restarts the debounce.
|
|
19607
|
+
const coordinationCondition = !remoteSetup && noticeReason ? tab.key : null;
|
|
17911
19608
|
useEffect(() => {
|
|
17912
19609
|
setVisibleCoordinationCondition(null);
|
|
17913
19610
|
if (coordinationCondition === null) return;
|
|
@@ -17919,10 +19616,21 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17919
19616
|
}, [coordinationCondition]);
|
|
17920
19617
|
const showCoordinationNotice = coordinationCondition !== null &&
|
|
17921
19618
|
visibleCoordinationCondition === coordinationCondition;
|
|
17922
|
-
const
|
|
17923
|
-
|
|
17924
|
-
|
|
17925
|
-
|
|
19619
|
+
const noticeResource = activeResource;
|
|
19620
|
+
const noticeSourceAt = collectorSourceAgeRef.current[tab.key]
|
|
19621
|
+
? now.getTime() - collectorSourceAgeRef.current[tab.key].ageMs -
|
|
19622
|
+
Math.max(0, performance.now() - collectorSourceAgeRef.current[tab.key].measuredAt)
|
|
19623
|
+
: lastOkRef.current[tab.key] ?? null;
|
|
19624
|
+
const noticeCause = activeRecovery ?? (noticeReason ? recoveryCause({
|
|
19625
|
+
reason: noticeReason === "other" ? "network-outage" : noticeReason,
|
|
19626
|
+
resource: noticeResource, sourceAt: noticeSourceAt,
|
|
19627
|
+
retryAt: activeGovernorDecision?.retryAt ?? activeGovernorDecision?.resetMs ??
|
|
19628
|
+
activeGovernorDecision?.notBefore ?? null,
|
|
19629
|
+
origin: tabError ? "source" : "coordination", tab: tabError ? tab.key : null,
|
|
19630
|
+
}) : null);
|
|
19631
|
+
const noticeLines = showCoordinationNotice && noticeCause
|
|
19632
|
+
? presentRecovery(noticeCause, { nowMs: now.getTime(), cols })
|
|
19633
|
+
: [!remoteSetup && displayError ? displayError : ""];
|
|
17926
19634
|
const noticeTone = showCoordinationNotice ? ATTENTION : displayError ? ERROR_TEXT : undefined;
|
|
17927
19635
|
const spin = SPINNER[frame % SPINNER.length];
|
|
17928
19636
|
|
|
@@ -17999,6 +19707,22 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
17999
19707
|
});
|
|
18000
19708
|
}
|
|
18001
19709
|
|
|
19710
|
+
if (runtime.background === "on-demand" && tab.key !== "actions" && data[tab.key] !== null &&
|
|
19711
|
+
["watching", "shared", "stale"].includes(semanticStatus.kind) && !tabError &&
|
|
19712
|
+
!activeFailure && !(tab.key === "security" && securityBlind)) {
|
|
19713
|
+
semanticStatus = refreshStatus({ onDemand: true });
|
|
19714
|
+
}
|
|
19715
|
+
|
|
19716
|
+
// A settled on-demand view has no data wake to advance its visible age.
|
|
19717
|
+
// Tick only its display clock; source timestamps and acquisition stay idle.
|
|
19718
|
+
const showCachedAge = semanticStatus.kind === "cached" && staleFor !== null;
|
|
19719
|
+
useEffect(() => {
|
|
19720
|
+
if (!showCachedAge) return;
|
|
19721
|
+
setNow(new Date());
|
|
19722
|
+
const timer = setInterval(() => setNow(new Date()), 1000);
|
|
19723
|
+
return () => clearInterval(timer);
|
|
19724
|
+
}, [showCachedAge]);
|
|
19725
|
+
|
|
18002
19726
|
const allItems = items ?? [];
|
|
18003
19727
|
const tabOffsetRaw = offset[tab.key] ?? 0;
|
|
18004
19728
|
const selectedKey = selected[tab.key] ?? null;
|
|
@@ -18199,7 +19923,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
18199
19923
|
!showHelp &&
|
|
18200
19924
|
tooNarrow &&
|
|
18201
19925
|
e(Text, { dimColor: true, wrap: "truncate-end" }, "too narrow"),
|
|
18202
|
-
|
|
19926
|
+
...noticeLines.map((line, index) =>
|
|
19927
|
+
e(Text, { key: `recovery:${index}`, color: noticeTone, wrap: "truncate-end" }, line)),
|
|
18203
19928
|
!showHelp &&
|
|
18204
19929
|
!tooNarrow &&
|
|
18205
19930
|
!remoteSetup &&
|
|
@@ -18294,7 +20019,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
18294
20019
|
status: semanticStatus,
|
|
18295
20020
|
detail: activeGovernorDecision,
|
|
18296
20021
|
spin: showSpinner ? spin : null,
|
|
18297
|
-
stale:
|
|
20022
|
+
stale: semanticStatus.kind === "cached" && staleFor != null
|
|
20023
|
+
? formatDuration(Math.max(0, staleFor)) : staleLabel,
|
|
18298
20024
|
nowMs: now.getTime(),
|
|
18299
20025
|
interactive,
|
|
18300
20026
|
cols: frameCols,
|
|
@@ -18589,6 +20315,7 @@ export {
|
|
|
18589
20315
|
resolveEffectiveCredential,
|
|
18590
20316
|
identityRegistryRoot,
|
|
18591
20317
|
inspectIdentityRegistry,
|
|
20318
|
+
doctorCachedQuotaScope,
|
|
18592
20319
|
claimIdentityBootstrap,
|
|
18593
20320
|
finishIdentityBootstrap,
|
|
18594
20321
|
createIdentityCoordinator,
|
|
@@ -18628,11 +20355,17 @@ export {
|
|
|
18628
20355
|
GOVERNOR_PHASE_WINDOW_MS,
|
|
18629
20356
|
BUDGET_PROBE_MS,
|
|
18630
20357
|
MIN_SAMPLE_CALLS,
|
|
20358
|
+
MAX_PACING_EXTERNAL_FACTOR,
|
|
20359
|
+
IDENTITY_UNCERTAIN_MAX_MS,
|
|
18631
20360
|
REQUEST_PRIORITIES,
|
|
18632
20361
|
normalizeBudgetResource,
|
|
18633
20362
|
budgetEpoch,
|
|
18634
20363
|
resourceReserve,
|
|
18635
20364
|
availableForGrant,
|
|
20365
|
+
governorChargedCost,
|
|
20366
|
+
governorAvailableForGrant,
|
|
20367
|
+
governorResourceDecision,
|
|
20368
|
+
governorResourceHeld,
|
|
18636
20369
|
nextExternalFactor,
|
|
18637
20370
|
resourceDecision,
|
|
18638
20371
|
governorPhaseOffset,
|
|
@@ -18658,6 +20391,7 @@ export {
|
|
|
18658
20391
|
serializeGovernorState,
|
|
18659
20392
|
readGovernorState,
|
|
18660
20393
|
writeGovernorState,
|
|
20394
|
+
writeGovernorQuotaState,
|
|
18661
20395
|
pidIsDead,
|
|
18662
20396
|
claimGovernorLock,
|
|
18663
20397
|
releaseGovernorLock,
|
|
@@ -18685,6 +20419,9 @@ export {
|
|
|
18685
20419
|
readIntentDecision,
|
|
18686
20420
|
cancelIntent,
|
|
18687
20421
|
startReservation,
|
|
20422
|
+
issueGovernorDispatch,
|
|
20423
|
+
terminalGovernorDispatch,
|
|
20424
|
+
acknowledgeSealedGeneration,
|
|
18688
20425
|
completeReservation,
|
|
18689
20426
|
settleReservationWithBudgetObservations,
|
|
18690
20427
|
recordResourceBlock,
|
|
@@ -18846,7 +20583,9 @@ export {
|
|
|
18846
20583
|
acquisitionRequestMetrics,
|
|
18847
20584
|
acquisitionFailureHold,
|
|
18848
20585
|
setRuntimeAcquisitionHold,
|
|
20586
|
+
clearScheduledRuntimeControlHold,
|
|
18849
20587
|
loadAcquisitionStore,
|
|
20588
|
+
readAcquisitionSnapshot,
|
|
18850
20589
|
createAcquisitionEngine,
|
|
18851
20590
|
pollPolicyInterval,
|
|
18852
20591
|
advanceUnchangedCount,
|
|
@@ -18882,11 +20621,27 @@ export {
|
|
|
18882
20621
|
runtimeIntentGate,
|
|
18883
20622
|
rateLimitBlockDecision,
|
|
18884
20623
|
coordinationNotice,
|
|
20624
|
+
recoveryCause,
|
|
20625
|
+
shouldClearRecoveryCause,
|
|
20626
|
+
recoveryRetryAt,
|
|
20627
|
+
terminalizationRecoveryCode,
|
|
20628
|
+
terminalizationBacklogForResource,
|
|
20629
|
+
receiptCapabilityFromReservation,
|
|
20630
|
+
requestFailureRecoveryCode,
|
|
20631
|
+
requestFailureRecovery,
|
|
20632
|
+
wallClockRecovery,
|
|
20633
|
+
collectorSourceAdvanced,
|
|
20634
|
+
cleanupQueueForResource,
|
|
20635
|
+
presentRecovery,
|
|
20636
|
+
recordRecoveryTransition,
|
|
20637
|
+
safeRecordRecoveryTransition,
|
|
20638
|
+
readRecoveryTransitions,
|
|
18885
20639
|
mergeRateLimitBlockPublications,
|
|
18886
20640
|
hydrateRateLimitBlockPublication,
|
|
18887
20641
|
retryRateLimitBlockPublication,
|
|
18888
20642
|
rateLimitBlockProbeRecovered,
|
|
18889
20643
|
admitGovernorOperation,
|
|
20644
|
+
queueTerminalization,
|
|
18890
20645
|
runAdmittedOperation,
|
|
18891
20646
|
abortableDelay,
|
|
18892
20647
|
awaitCollectorReservation,
|
|
@@ -18912,6 +20667,7 @@ export {
|
|
|
18912
20667
|
demandedPageCount,
|
|
18913
20668
|
mergeDemandedPages,
|
|
18914
20669
|
operationPausedUntil,
|
|
20670
|
+
pauseOperation,
|
|
18915
20671
|
LIST_LIMIT,
|
|
18916
20672
|
readGraphqlObserver,
|
|
18917
20673
|
GRAPHQL_QUERIES,
|