gh-glance 0.11.2 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +112 -1
- package/README.md +36 -13
- package/index.mjs +2104 -385
- package/package.json +2 -2
package/index.mjs
CHANGED
|
@@ -23,6 +23,7 @@
|
|
|
23
23
|
process.env.NODE_ENV ??= "production";
|
|
24
24
|
|
|
25
25
|
import { execFile, spawn } from "node:child_process";
|
|
26
|
+
import { AsyncLocalStorage } from "node:async_hooks";
|
|
26
27
|
import { createHash, randomUUID } from "node:crypto";
|
|
27
28
|
import {
|
|
28
29
|
chmodSync,
|
|
@@ -43,6 +44,7 @@ import { fileURLToPath } from "node:url";
|
|
|
43
44
|
import { promisify } from "node:util";
|
|
44
45
|
|
|
45
46
|
const execFileAsync = promisify(execFile);
|
|
47
|
+
const requestIdentityStorage = new AsyncLocalStorage();
|
|
46
48
|
|
|
47
49
|
// Running as the CLI vs. being imported by a test. Everything with a side
|
|
48
50
|
// effect -- argv handling, the TTY guard, the preflight, entering the alternate
|
|
@@ -616,6 +618,7 @@ function registerLiveAbort(controller) {
|
|
|
616
618
|
liveAbort = controller;
|
|
617
619
|
}
|
|
618
620
|
function abortLiveRequests() {
|
|
621
|
+
runtimeIdentityCoordinator?.close();
|
|
619
622
|
liveAbort?.abort();
|
|
620
623
|
liveAbort = null;
|
|
621
624
|
}
|
|
@@ -627,24 +630,133 @@ function forwardSignalToChild(child, signal) {
|
|
|
627
630
|
return child.kill(signal);
|
|
628
631
|
}
|
|
629
632
|
|
|
630
|
-
|
|
633
|
+
function inspectAdmittedHttpStart(scope, operation, now = Date.now) {
|
|
634
|
+
const costs = operationCost(operation);
|
|
635
|
+
if (!costs) return { ok: false, reason: "undeclared" };
|
|
636
|
+
if (RATE_RESOURCES.every((resource) => costs[resource] === 0)) return { ok: true };
|
|
637
|
+
const readNow = typeof now === "function" ? now : () => now;
|
|
638
|
+
const snapshot = inspectGovernor(scope, readNow());
|
|
639
|
+
if (!snapshot.ok || snapshot.missing) return snapshot.ok ? { ok: false, reason: "corrupt" } : snapshot;
|
|
640
|
+
const checkedAt = readNow();
|
|
641
|
+
for (const resource of RATE_RESOURCES) {
|
|
642
|
+
if (costs[resource] <= 0) continue;
|
|
643
|
+
const budget = snapshot.value.budgets[resource];
|
|
644
|
+
if (!budget) return { ok: false, reason: "budget-unknown", resource };
|
|
645
|
+
if (budget.blockUntil > checkedAt) return { ok: false, reason: "blocked", resource, retryAt: budget.blockUntil };
|
|
646
|
+
if (budget.resetMs <= checkedAt) return { ok: false, reason: "budget-reset", resource, retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
|
|
647
|
+
}
|
|
648
|
+
return { ok: true };
|
|
649
|
+
}
|
|
650
|
+
|
|
651
|
+
// Re-checked either side of every wait a request can sit in, because admission,
|
|
652
|
+
// the transport permit and the subprocess boundary are each far enough apart
|
|
653
|
+
// that the account can change in between. One definition so the three checks
|
|
654
|
+
// cannot drift, and so a request costs one identity snapshot instead of three.
|
|
655
|
+
function assertBoundCredential(bound) {
|
|
656
|
+
if (bound?.accessKey && bound.accessKey !== runtimeIdentityCoordinator?.current()?.accessKey) {
|
|
657
|
+
throw new Error("Credential changed before request start");
|
|
658
|
+
}
|
|
659
|
+
}
|
|
660
|
+
|
|
661
|
+
// GraphQL documents are supplied on stdin rather than as an argument, so a
|
|
662
|
+
// query never lands in argv where `ps` or an error string could carry it, and
|
|
663
|
+
// so its size is bounded by a pipe rather than by the platform argument limit.
|
|
664
|
+
// promisify(execFile) exposes the child, which is what lets stdin be written
|
|
665
|
+
// without giving up the timeout, kill signal, maxBuffer, abort and redaction
|
|
666
|
+
// contracts every other call depends on.
|
|
667
|
+
// The control plane cannot be gated on the capacity it exists to establish.
|
|
668
|
+
// Both observers are exempt from the data-admission recheck for that reason:
|
|
669
|
+
// a resource whose budget is unknown would otherwise refuse the one request
|
|
670
|
+
// that could learn it, and the tab would wait for a number nothing can fetch.
|
|
671
|
+
// They remain bounded elsewhere -- by the rolling attempt allowance, the shared
|
|
672
|
+
// transport permit and the secondary cooldown -- so this is an exemption from
|
|
673
|
+
// data admission, not from accounting.
|
|
674
|
+
const CONTROL_OPERATIONS = ["budget-core-observer", "graphql-observer"];
|
|
675
|
+
|
|
676
|
+
// An operation whose actual cost exceeded its declared bound is paused until its
|
|
677
|
+
// resource resets. Continuing to spend against a bound already known to be wrong
|
|
678
|
+
// is how a reserve gets crossed with every individual request looking admissible.
|
|
679
|
+
// Process-local on purpose: the bound is a property of this build's query text,
|
|
680
|
+
// so a differently-built pane's bound is not this one's to suspend.
|
|
681
|
+
const pausedOperations = new Map();
|
|
682
|
+
|
|
683
|
+
function pauseOperation(operation, untilMs) {
|
|
684
|
+
if (!Number.isFinite(untilMs)) return;
|
|
685
|
+
pausedOperations.set(operation, Math.max(pausedOperations.get(operation) ?? 0, untilMs));
|
|
686
|
+
}
|
|
687
|
+
|
|
688
|
+
function operationPausedUntil(operation, now = Date.now()) {
|
|
689
|
+
const until = pausedOperations.get(operation);
|
|
690
|
+
if (until === undefined) return null;
|
|
691
|
+
if (until <= now) { pausedOperations.delete(operation); return null; }
|
|
692
|
+
return until;
|
|
693
|
+
}
|
|
694
|
+
|
|
695
|
+
async function runGh(args, { signal, operation, input = null } = {}) {
|
|
631
696
|
if (operationCost(operation) === null) {
|
|
632
697
|
throw new Error(`undeclared gh operation: ${operation ?? "missing"}`);
|
|
633
698
|
}
|
|
699
|
+
const bound = requestIdentityStorage.getStore();
|
|
700
|
+
assertBoundCredential(bound);
|
|
701
|
+
const local = ["version", "local-git"].includes(operation);
|
|
702
|
+
let control = null;
|
|
703
|
+
const permit = runtimeIdentityCoordinator && !local
|
|
704
|
+
? await acquireIdentityHttpPermit(runtimeIdentityCoordinator, { signal }) : null;
|
|
634
705
|
const startedAt = Date.now();
|
|
706
|
+
let requestError = null;
|
|
707
|
+
let requestStdout = null;
|
|
635
708
|
try {
|
|
636
|
-
|
|
709
|
+
assertBoundCredential(bound);
|
|
710
|
+
if (runtimeIdentityCoordinator && operation === "budget-core-observer") {
|
|
711
|
+
control = startIdentityControl(runtimeIdentityCoordinator);
|
|
712
|
+
if (!control.ok) throw new Error(identityCoordinationMessage(control.reason));
|
|
713
|
+
}
|
|
714
|
+
// Admission can precede this per-call transport slot by many seconds.
|
|
715
|
+
// A sibling request may publish a shared hold while this call is queued.
|
|
716
|
+
// Keep the started envelope charged, but recheck resource validity at the
|
|
717
|
+
// actual subprocess boundary without reserving the same work again.
|
|
718
|
+
const pausedUntil = operationPausedUntil(operation);
|
|
719
|
+
if (pausedUntil !== null) {
|
|
720
|
+
throw new Error(`Operation paused after an unbounded cost (retry after ${new Date(pausedUntil).toISOString()})`);
|
|
721
|
+
}
|
|
722
|
+
if (bound && !local && !CONTROL_OPERATIONS.includes(operation)) {
|
|
723
|
+
const ready = inspectAdmittedHttpStart(bound, operation);
|
|
724
|
+
if (!ready.ok) throw new Error(`API budget paused (${ready.reason})`);
|
|
725
|
+
}
|
|
726
|
+
assertBoundCredential(bound);
|
|
727
|
+
const pending = execFileAsync("gh", args, {
|
|
637
728
|
timeout: GH_TIMEOUT_MS,
|
|
638
729
|
killSignal: "SIGKILL",
|
|
639
730
|
maxBuffer: GH_MAX_BUFFER,
|
|
640
731
|
env: { ...process.env, ...GH_ENV_OVERRIDES },
|
|
641
732
|
signal,
|
|
642
733
|
});
|
|
734
|
+
if (typeof input === "string") {
|
|
735
|
+
// A child killed by the timeout or the abort signal closes stdin under
|
|
736
|
+
// us; that EPIPE is the kill's consequence, not the failure worth
|
|
737
|
+
// reporting, so let the awaited result below carry the real error.
|
|
738
|
+
pending.child.stdin?.on("error", () => {});
|
|
739
|
+
pending.child.stdin?.end(input);
|
|
740
|
+
}
|
|
741
|
+
const { stdout } = await pending;
|
|
742
|
+
requestStdout = stdout;
|
|
643
743
|
logGh(args, startedAt, `ok ${stdout.length}B`);
|
|
644
744
|
return stdout;
|
|
645
745
|
} catch (err) {
|
|
746
|
+
requestError = err;
|
|
646
747
|
logGh(args, startedAt, `FAILED ${shortErr(err)}`);
|
|
647
748
|
throw err;
|
|
749
|
+
} finally {
|
|
750
|
+
if (permit) {
|
|
751
|
+
const release = () => releaseIdentityHttpPermit(runtimeIdentityCoordinator, permit, requestError);
|
|
752
|
+
const released = await retryIdentityCompletion(release, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !runtimeIdentityCoordinator.isClosed() });
|
|
753
|
+
if (!released.ok) runtimeIdentityCoordinator.deferCompletion(`permit:${permit.nonce}`, release);
|
|
754
|
+
}
|
|
755
|
+
if (control?.ok) {
|
|
756
|
+
const settle = () => settleIdentityControl(runtimeIdentityCoordinator, control.value, requestStdout ?? requestError?.stdout);
|
|
757
|
+
const settled = await retryIdentityCompletion(settle, { timeoutMs: IDENTITY_RELEASE_WAIT_MS, shouldRetry: () => !runtimeIdentityCoordinator.isClosed() });
|
|
758
|
+
if (!settled.ok) runtimeIdentityCoordinator.deferCompletion(`control:${control.value.id}`, settle);
|
|
759
|
+
}
|
|
648
760
|
}
|
|
649
761
|
}
|
|
650
762
|
|
|
@@ -734,14 +846,141 @@ async function ghApi(args, { operation, signal, etag = null, run = runGh } = {})
|
|
|
734
846
|
}
|
|
735
847
|
}
|
|
736
848
|
|
|
737
|
-
//
|
|
738
|
-
|
|
739
|
-
//
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
849
|
+
// ---------- Explicit GraphQL ----------
|
|
850
|
+
|
|
851
|
+
// The version travels in the query key, so a changed document invalidates its
|
|
852
|
+
// cursors instead of paging a new shape with an old cursor.
|
|
853
|
+
const GRAPHQL_QUERY_VERSION = 1;
|
|
854
|
+
// GitHub prices a connection by the pages it could return, and the row's single
|
|
855
|
+
// label is a nested connection of its own. Two points is the conservative bound
|
|
856
|
+
// for one page; the observer selects nothing but the meter and costs one.
|
|
857
|
+
const GRAPHQL_PAGE_POINTS = 2;
|
|
858
|
+
const GRAPHQL_OBSERVER_POINTS = 1;
|
|
859
|
+
// 50 rows per page and no more than three pages preserves the existing
|
|
860
|
+
// LIST_LIMIT of 150 while making every page a separately admitted request.
|
|
861
|
+
const GRAPHQL_PAGE_SIZE = 50;
|
|
862
|
+
// Two hours: comfortably past any real primary window, and short enough that a
|
|
863
|
+
// nonsense timestamp cannot park the budget in a reset that never arrives.
|
|
864
|
+
const GRAPHQL_MAX_RESET_MS = 2 * 60 * 60 * 1000;
|
|
865
|
+
|
|
866
|
+
// Issues and pull requests are deliberately two documents, not one with two
|
|
867
|
+
// connections: a combined query cannot publish either tab until both halves
|
|
868
|
+
// are complete, which is exactly the completion barrier this phase removes.
|
|
869
|
+
// Both select only what a row renders, which is also what keeps the cost at
|
|
870
|
+
// its declared bound.
|
|
871
|
+
const ISSUE_PAGE_QUERY = `query($owner:String!,$name:String!,$first:Int!,$after:String){
|
|
872
|
+
repository(owner:$owner,name:$name){
|
|
873
|
+
id
|
|
874
|
+
name
|
|
875
|
+
nameWithOwner
|
|
876
|
+
url
|
|
877
|
+
issues(first:$first,after:$after,states:OPEN,orderBy:{field:UPDATED_AT,direction:DESC}){
|
|
878
|
+
totalCount
|
|
879
|
+
pageInfo{hasNextPage endCursor}
|
|
880
|
+
nodes{id number title url updatedAt author{login} labels(first:1){nodes{name}}}
|
|
881
|
+
}
|
|
882
|
+
}
|
|
883
|
+
rateLimit{cost limit used remaining resetAt}
|
|
884
|
+
}`;
|
|
885
|
+
|
|
886
|
+
const PULL_PAGE_QUERY = `query($owner:String!,$name:String!,$first:Int!,$after:String){
|
|
887
|
+
repository(owner:$owner,name:$name){
|
|
888
|
+
id
|
|
889
|
+
name
|
|
890
|
+
nameWithOwner
|
|
891
|
+
url
|
|
892
|
+
pullRequests(first:$first,after:$after,states:OPEN,orderBy:{field:UPDATED_AT,direction:DESC}){
|
|
893
|
+
totalCount
|
|
894
|
+
pageInfo{hasNextPage endCursor}
|
|
895
|
+
nodes{id number title url updatedAt author{login} headRefName isDraft reviewDecision}
|
|
896
|
+
}
|
|
897
|
+
}
|
|
898
|
+
rateLimit{cost limit used remaining resetAt}
|
|
899
|
+
}`;
|
|
900
|
+
|
|
901
|
+
// Selects the meter and nothing else. This is the only claimed GraphQL
|
|
902
|
+
// observer; a data page's counters constrain the same epoch but never open a
|
|
903
|
+
// new one.
|
|
904
|
+
const GRAPHQL_OBSERVER_QUERY = `query{rateLimit{cost limit used remaining resetAt}}`;
|
|
905
|
+
|
|
906
|
+
// Replaces `gh repo view --json`, whose GraphQL cost was real but undeclared.
|
|
907
|
+
const REPOSITORY_QUERY = `query($owner:String!,$name:String!){
|
|
908
|
+
repository(owner:$owner,name:$name){
|
|
909
|
+
id
|
|
910
|
+
nameWithOwner
|
|
911
|
+
url
|
|
912
|
+
viewerPermission
|
|
913
|
+
}
|
|
914
|
+
rateLimit{cost limit used remaining resetAt}
|
|
915
|
+
}`;
|
|
916
|
+
|
|
917
|
+
const GRAPHQL_QUERIES = Object.freeze({
|
|
918
|
+
repository: { query: REPOSITORY_QUERY, connection: null, points: GRAPHQL_OBSERVER_POINTS },
|
|
919
|
+
issues: { query: ISSUE_PAGE_QUERY, connection: "issues", points: GRAPHQL_PAGE_POINTS },
|
|
920
|
+
prs: { query: PULL_PAGE_QUERY, connection: "pullRequests", points: GRAPHQL_PAGE_POINTS },
|
|
921
|
+
observer: { query: GRAPHQL_OBSERVER_QUERY, connection: null, points: GRAPHQL_OBSERVER_POINTS },
|
|
922
|
+
});
|
|
923
|
+
|
|
924
|
+
// `--input -` keeps the document and its variables on stdin as one typed JSON
|
|
925
|
+
// object, so variables stay typed rather than becoming the strings that
|
|
926
|
+
// `-f`/`-F` would produce, and nothing reaches argv.
|
|
927
|
+
function graphqlArgs(host = effectiveRuntimeHost()) {
|
|
928
|
+
return ["api", "-i", "graphql", "--input", "-", ...apiHostArgs(host)];
|
|
743
929
|
}
|
|
744
930
|
|
|
931
|
+
function graphqlInput(kind, variables) {
|
|
932
|
+
const declared = pick(GRAPHQL_QUERIES, kind, null);
|
|
933
|
+
if (!declared) throw new Error(`undeclared graphql query: ${kind}`);
|
|
934
|
+
if (variables === null) throw new Error(UNRESOLVED_REPOSITORY);
|
|
935
|
+
return JSON.stringify({ query: declared.query, variables });
|
|
936
|
+
}
|
|
937
|
+
|
|
938
|
+
// `gh` resolves an omitted repository from the working directory. GraphQL has
|
|
939
|
+
// no such inference, so the slug must be resolved here or the document cannot
|
|
940
|
+
// be built at all -- and running without `--repo` is the documented default.
|
|
941
|
+
//
|
|
942
|
+
// Refusing is deliberate. A half-filled variable set is worse than no request:
|
|
943
|
+
// GitHub answers a missing `String!` with HTTP 200 and an errors array, which
|
|
944
|
+
// is indistinguishable from a denied query, so the tab would report a
|
|
945
|
+
// permission problem for what is really an unresolved target.
|
|
946
|
+
function graphqlRepositoryVariables(repository = effectiveRuntimeRepository()) {
|
|
947
|
+
if (!repository) return null;
|
|
948
|
+
const [owner, name] = repository.split("/");
|
|
949
|
+
return { owner, name };
|
|
950
|
+
}
|
|
951
|
+
|
|
952
|
+
function graphqlPageVariables(after = null, repository = effectiveRuntimeRepository()) {
|
|
953
|
+
const target = graphqlRepositoryVariables(repository);
|
|
954
|
+
return target && { ...target, first: GRAPHQL_PAGE_SIZE, after };
|
|
955
|
+
}
|
|
956
|
+
|
|
957
|
+
// GitHub answers a rejected query with HTTP 200 and an `errors` array, and can
|
|
958
|
+
// answer a partly-resolvable one with both `data` and `errors`. Headers and
|
|
959
|
+
// envelope are therefore read separately: budget evidence stays usable even
|
|
960
|
+
// when the data is not, which is the only way a failed page can still be paid
|
|
961
|
+
// for honestly.
|
|
962
|
+
function parseGraphqlEnvelope(body) {
|
|
963
|
+
let envelope;
|
|
964
|
+
try { envelope = JSON.parse(body); } catch { return { ok: false, reason: "unparseable", errors: [], data: null, rateLimit: null }; }
|
|
965
|
+
if (!isRecord(envelope)) return { ok: false, reason: "unparseable", errors: [], data: null, rateLimit: null };
|
|
966
|
+
const errors = Array.isArray(envelope.errors) ? envelope.errors : [];
|
|
967
|
+
const data = isRecord(envelope.data) ? envelope.data : null;
|
|
968
|
+
const meter = isRecord(data?.rateLimit) ? data.rateLimit : null;
|
|
969
|
+
const number = (value) => (Number.isSafeInteger(value) && value >= 0 ? value : null);
|
|
970
|
+
// A primary window is an hour; anything beyond a generous ceiling is not a
|
|
971
|
+
// reset this app can wait for, and accepting it would pin an epoch that
|
|
972
|
+
// `budget.resetMs <= now` never clears.
|
|
973
|
+
const parsedReset = typeof meter?.resetAt === "string" ? Date.parse(meter.resetAt) : Number.NaN;
|
|
974
|
+
const resetMs = Number.isFinite(parsedReset) && parsedReset <= Date.now() + GRAPHQL_MAX_RESET_MS
|
|
975
|
+
? parsedReset
|
|
976
|
+
: Number.NaN;
|
|
977
|
+
const rateLimit = meter && [number(meter.limit), number(meter.used), number(meter.remaining)].every((value) => value !== null) && Number.isFinite(resetMs)
|
|
978
|
+
? { resource: "graphql", limit: meter.limit, used: meter.used, remaining: meter.remaining, resetMs, cost: number(meter.cost) }
|
|
979
|
+
: null;
|
|
980
|
+
return { ok: errors.length === 0 && data !== null, reason: errors.length > 0 ? "graphql-errors" : data === null ? "no-data" : null, errors, data, rateLimit };
|
|
981
|
+
}
|
|
982
|
+
|
|
983
|
+
|
|
745
984
|
function normalizeHost(value) {
|
|
746
985
|
if (typeof value !== "string") return null;
|
|
747
986
|
const host = value.trim().toLowerCase();
|
|
@@ -787,6 +1026,51 @@ function resolveEffectiveHost({
|
|
|
787
1026
|
return hosts.size === 1 ? [...hosts][0] : null;
|
|
788
1027
|
}
|
|
789
1028
|
|
|
1029
|
+
const UNRESOLVED_REPOSITORY = "Repository could not be resolved; pass --repo owner/name";
|
|
1030
|
+
|
|
1031
|
+
// The owner/name half of the same question resolveEffectiveHost answers for the
|
|
1032
|
+
// host, and in the same precedence: an explicit --repo, then GH_REPO, then an
|
|
1033
|
+
// unambiguous git remote. Ambiguity resolves to null rather than to a guess.
|
|
1034
|
+
function remoteSlug(value) {
|
|
1035
|
+
if (typeof value !== "string" || value.length === 0) return null;
|
|
1036
|
+
let path;
|
|
1037
|
+
if (value.includes("://")) {
|
|
1038
|
+
try { path = new URL(value).pathname; } catch { return null; }
|
|
1039
|
+
} else {
|
|
1040
|
+
path = /^(?:[^@\s]+@)?[^:/\s]+:([^\s]+)$/.exec(value)?.[1] ?? null;
|
|
1041
|
+
}
|
|
1042
|
+
if (path === null) return null;
|
|
1043
|
+
// Trimmed by index rather than with /\/+$/. An end-anchored quantifier is
|
|
1044
|
+
// retried from every position, so a remote URL carrying a long run of slashes
|
|
1045
|
+
// costs quadratic time to reject (CodeQL js/polynomial-redos, high). Remote
|
|
1046
|
+
// URLs are library input: `git remote -v` reports whatever is configured.
|
|
1047
|
+
let start = 0;
|
|
1048
|
+
let end = path.length;
|
|
1049
|
+
while (start < end && path[start] === "/") start += 1;
|
|
1050
|
+
while (end > start && path[end - 1] === "/") end -= 1;
|
|
1051
|
+
const trimmed = path.slice(start, end);
|
|
1052
|
+
const slug = trimmed.endsWith(".git") ? trimmed.slice(0, -4) : trimmed;
|
|
1053
|
+
return REPO_PATTERN.test(slug) ? slug : null;
|
|
1054
|
+
}
|
|
1055
|
+
|
|
1056
|
+
function resolveEffectiveRepository({ runtimeRepo = null, ghRepo = null, remoteUrls = [] } = {}) {
|
|
1057
|
+
if (runtimeRepo) return runtimeRepo;
|
|
1058
|
+
if (ghRepo) {
|
|
1059
|
+
try { return parseRepoTarget(ghRepo).slug; } catch { return null; }
|
|
1060
|
+
}
|
|
1061
|
+
const slugs = new Set(remoteUrls.map(remoteSlug).filter(Boolean));
|
|
1062
|
+
return slugs.size === 1 ? [...slugs][0] : null;
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
function effectiveRuntimeRepository(options = {}) {
|
|
1066
|
+
return resolveEffectiveRepository({
|
|
1067
|
+
runtimeRepo: runtime.repo,
|
|
1068
|
+
ghRepo: process.env.GH_REPO,
|
|
1069
|
+
remoteUrls: runtimeRemoteUrls,
|
|
1070
|
+
...options,
|
|
1071
|
+
});
|
|
1072
|
+
}
|
|
1073
|
+
|
|
790
1074
|
function effectiveRuntimeHost(options = {}) {
|
|
791
1075
|
return resolveEffectiveHost({
|
|
792
1076
|
runtimeHost: runtime.host,
|
|
@@ -799,11 +1083,6 @@ function effectiveRuntimeHost(options = {}) {
|
|
|
799
1083
|
});
|
|
800
1084
|
}
|
|
801
1085
|
|
|
802
|
-
// `--repo` for the list subcommands. Empty when unset, so the argv vector stays
|
|
803
|
-
// byte-identical to what shipped before.
|
|
804
|
-
function repoArgs() {
|
|
805
|
-
return runtime.repo ? ["--repo", qualifiedRepo()] : [];
|
|
806
|
-
}
|
|
807
1086
|
|
|
808
1087
|
// `gh api` has no --repo, so a host-qualified target has nowhere to put its
|
|
809
1088
|
// host: GH_REPO=host/owner/repo supplies the owner and repo and *ignores* the
|
|
@@ -831,32 +1110,12 @@ function apiPath(path) {
|
|
|
831
1110
|
// inline, because `--doctor` reports these vectors and a second copy of them
|
|
832
1111
|
// would be a report that drifts away from what the dashboard actually sends --
|
|
833
1112
|
// which is the failure mode the whole diagnostics command exists to rule out.
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
return [
|
|
837
|
-
"repo",
|
|
838
|
-
"view",
|
|
839
|
-
...(target ? [target] : []),
|
|
840
|
-
"--json",
|
|
841
|
-
"nameWithOwner,url,viewerPermission",
|
|
842
|
-
];
|
|
843
|
-
}
|
|
844
|
-
|
|
845
|
-
function authContextArgs() {
|
|
846
|
-
return [
|
|
847
|
-
"auth",
|
|
848
|
-
"status",
|
|
849
|
-
"--active",
|
|
850
|
-
"--json",
|
|
851
|
-
"hosts",
|
|
852
|
-
"--jq",
|
|
853
|
-
".hosts | to_entries | map(.key as $host | .value[] | select(.active == true) | {host: $host, login: .login})",
|
|
854
|
-
];
|
|
855
|
-
}
|
|
856
|
-
|
|
1113
|
+
// Parses the explicit repository query's envelope. The shape the rest of the
|
|
1114
|
+
// failure context consumes is unchanged; only how it was obtained is.
|
|
857
1115
|
function parseRepoContext(raw) {
|
|
858
1116
|
try {
|
|
859
|
-
const
|
|
1117
|
+
const envelope = JSON.parse(raw);
|
|
1118
|
+
const value = isRecord(envelope?.data?.repository) ? envelope.data.repository : envelope;
|
|
860
1119
|
if (
|
|
861
1120
|
value == null ||
|
|
862
1121
|
Array.isArray(value) ||
|
|
@@ -927,15 +1186,17 @@ async function resolveFailureContext(signal, governor = null, { run = runGh } =
|
|
|
927
1186
|
...governor,
|
|
928
1187
|
operation: "failure-context:repository",
|
|
929
1188
|
signal,
|
|
930
|
-
run: (admittedSignal) => run(
|
|
1189
|
+
run: (admittedSignal) => run(graphqlArgs(), {
|
|
931
1190
|
signal: admittedSignal,
|
|
932
1191
|
operation: "failure-context:repository",
|
|
933
|
-
|
|
1192
|
+
input: graphqlInput("repository", graphqlRepositoryVariables()),
|
|
1193
|
+
}).then((stdout) => parseGhApiResponse(stdout).body),
|
|
934
1194
|
}).then((result) => result.ok ? result.value : Promise.reject(result.error))
|
|
935
1195
|
: Promise.reject(new Error("API budget unavailable"));
|
|
1196
|
+
const identity = runtimeIdentityCoordinator?.current();
|
|
936
1197
|
const [repo, auth] = await Promise.allSettled([
|
|
937
1198
|
repoCall,
|
|
938
|
-
|
|
1199
|
+
identity ? Promise.resolve(JSON.stringify([{ host: identity.host, login: identity.login }])) : Promise.reject(new Error("Verified identity unavailable")),
|
|
939
1200
|
]);
|
|
940
1201
|
return buildFailureContext(repo, auth);
|
|
941
1202
|
}
|
|
@@ -987,7 +1248,12 @@ function actionsRunsArgs(limit) {
|
|
|
987
1248
|
apiPath(`repos/{owner}/{repo}/actions/runs?exclude_pull_requests=true&per_page=${limit}`),
|
|
988
1249
|
...apiHostArgs(),
|
|
989
1250
|
"--jq",
|
|
990
|
-
|
|
1251
|
+
// html_url is projected because a run's page must come from the row itself.
|
|
1252
|
+
// Deriving it from the repository slug only works when --repo was given;
|
|
1253
|
+
// with an inferred repository the app never learns the slug, and the row is
|
|
1254
|
+
// the one place the answer is always present. It costs nothing extra: the
|
|
1255
|
+
// field is already in the response being parsed.
|
|
1256
|
+
"[.workflow_runs[] | {databaseId: .id, displayTitle: .display_title, number: .run_number, headBranch: .head_branch, status, conclusion, startedAt: .run_started_at, updatedAt: .updated_at, workflowId: .workflow_id, url: .html_url}]",
|
|
991
1257
|
];
|
|
992
1258
|
}
|
|
993
1259
|
|
|
@@ -1015,6 +1281,7 @@ function parseActionsBodies(runsBody, workflowsBody) {
|
|
|
1015
1281
|
conclusion: run.conclusion,
|
|
1016
1282
|
startedAt: run.startedAt,
|
|
1017
1283
|
updatedAt: run.updatedAt,
|
|
1284
|
+
url: safe(run.url ?? ""),
|
|
1018
1285
|
}));
|
|
1019
1286
|
}
|
|
1020
1287
|
|
|
@@ -1140,85 +1407,188 @@ async function fetchActions(limit, signal, {
|
|
|
1140
1407
|
};
|
|
1141
1408
|
}
|
|
1142
1409
|
|
|
1143
|
-
//
|
|
1144
|
-
//
|
|
1145
|
-
//
|
|
1146
|
-
//
|
|
1147
|
-
//
|
|
1148
|
-
//
|
|
1149
|
-
//
|
|
1150
|
-
|
|
1151
|
-
|
|
1410
|
+
// Ordering is part of the query now (`orderBy: UPDATED_AT DESC`) rather than a
|
|
1411
|
+
// `--search sort:` flag. The AGE column renders updatedAt, so a created-order
|
|
1412
|
+
// list made that column non-monotonic and, worse, truncating to pane height
|
|
1413
|
+
// dropped the oldest-*created* rows -- a PR opened six months ago and reviewed
|
|
1414
|
+
// five minutes ago was invisible. The old fix routed `gh issue list` through
|
|
1415
|
+
// GraphQL implicitly and at an unobservable cost; asking for the order directly
|
|
1416
|
+
// keeps the behaviour and makes the price visible.
|
|
1417
|
+
|
|
1418
|
+
// One page of an explicit connection. Headers and envelope are settled
|
|
1419
|
+
// separately so a rejected query still pays honestly: GitHub answers those with
|
|
1420
|
+
// HTTP 200 and an `errors` array, and its counters are trustworthy even when
|
|
1421
|
+
// its data is not.
|
|
1422
|
+
async function fetchGraphqlPage(kind, { signal, after = null, run = runGh, operation, host = effectiveRuntimeHost(), variables = null } = {}) {
|
|
1423
|
+
const declared = pick(GRAPHQL_QUERIES, kind, null);
|
|
1424
|
+
if (!declared) throw new Error(`undeclared graphql query: ${kind}`);
|
|
1425
|
+
let stdout;
|
|
1426
|
+
let failure = null;
|
|
1427
|
+
try {
|
|
1428
|
+
stdout = await run(graphqlArgs(host), { signal, operation, input: graphqlInput(kind, variables ?? graphqlPageVariables(after)) });
|
|
1429
|
+
} catch (error) {
|
|
1430
|
+
failure = error;
|
|
1431
|
+
// A failed call can still carry a complete response body: `gh` exits
|
|
1432
|
+
// non-zero on 4xx/5xx but has already written the headers and envelope,
|
|
1433
|
+
// and that is where trustworthy budget evidence lives.
|
|
1434
|
+
stdout = typeof error?.stdout === "string" ? error.stdout : "";
|
|
1435
|
+
}
|
|
1436
|
+
const parsed = parseGhApiResponse(stdout);
|
|
1437
|
+
const envelope = parseGraphqlEnvelope(parsed.body);
|
|
1438
|
+
// Prefer the envelope's meter: it carries this query's actual `cost`, which
|
|
1439
|
+
// the headers do not. Headers remain the fallback when the body is unusable.
|
|
1440
|
+
const meter = envelope.rateLimit ?? pickRateLimit(parsed.headers);
|
|
1441
|
+
// Absent evidence never refunds. An unobserved page keeps its conservative
|
|
1442
|
+
// reservation rather than being treated as free.
|
|
1443
|
+
const observedCost = Number.isSafeInteger(envelope.rateLimit?.cost)
|
|
1444
|
+
? envelope.rateLimit.cost
|
|
1445
|
+
: declared.points;
|
|
1446
|
+
const observation = meter && meter.resource === "graphql"
|
|
1447
|
+
? responseHeaderObservation(meter, observedCost)
|
|
1448
|
+
: null;
|
|
1449
|
+
return {
|
|
1450
|
+
kind,
|
|
1451
|
+
status: parsed.status,
|
|
1452
|
+
ok: failure === null && parsed.status === 200 && envelope.ok,
|
|
1453
|
+
reason: failure ? "transport" : parsed.status !== 200 ? "http" : envelope.reason,
|
|
1454
|
+
failure,
|
|
1455
|
+
data: envelope.data,
|
|
1456
|
+
// Recorded in full even when it exceeds the declared bound: an overrun is
|
|
1457
|
+
// evidence to reconcile, not a bookkeeping error to discard.
|
|
1458
|
+
observedCost,
|
|
1459
|
+
overrun: observedCost > declared.points,
|
|
1460
|
+
observations: observation ? [observation] : [],
|
|
1461
|
+
};
|
|
1462
|
+
}
|
|
1152
1463
|
|
|
1153
|
-
function
|
|
1154
|
-
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
...repoArgs(),
|
|
1158
|
-
"--state",
|
|
1159
|
-
"open",
|
|
1160
|
-
"--limit",
|
|
1161
|
-
String(LIST_LIMIT),
|
|
1162
|
-
...SORT_RECENT,
|
|
1163
|
-
"--json",
|
|
1164
|
-
"number,title,author,labels,createdAt,updatedAt",
|
|
1165
|
-
];
|
|
1464
|
+
function graphqlConnection(page, connection) {
|
|
1465
|
+
const repository = isRecord(page.data?.repository) ? page.data.repository : null;
|
|
1466
|
+
const value = isRecord(repository?.[connection]) ? repository[connection] : null;
|
|
1467
|
+
return value && Array.isArray(value.nodes) ? value : null;
|
|
1166
1468
|
}
|
|
1167
1469
|
|
|
1168
|
-
|
|
1169
|
-
|
|
1470
|
+
const ISSUE_ROW = (node) => ({
|
|
1471
|
+
number: node.number,
|
|
1472
|
+
title: safe(node.title),
|
|
1473
|
+
author: safe(node.author?.login ?? ""),
|
|
1474
|
+
label: safe(node.labels?.nodes?.[0]?.name ?? ""),
|
|
1475
|
+
updatedAt: node.updatedAt,
|
|
1476
|
+
url: safe(node.url ?? ""),
|
|
1477
|
+
});
|
|
1478
|
+
|
|
1479
|
+
const PR_ROW = (node) => ({
|
|
1480
|
+
number: node.number,
|
|
1481
|
+
title: safe(node.title),
|
|
1482
|
+
author: safe(node.author?.login ?? ""),
|
|
1483
|
+
headRefName: safe(node.headRefName),
|
|
1484
|
+
isDraft: node.isDraft,
|
|
1485
|
+
reviewDecision: node.reviewDecision,
|
|
1486
|
+
updatedAt: node.updatedAt,
|
|
1487
|
+
url: safe(node.url ?? ""),
|
|
1488
|
+
});
|
|
1489
|
+
|
|
1490
|
+
// Walks the connection one explicitly admitted page at a time. The first page
|
|
1491
|
+
// is covered by the tab's own reservation; each later page reserves its own
|
|
1492
|
+
// envelope, so a denial there is an ordinary scheduling outcome rather than a
|
|
1493
|
+
// hidden overspend. Rows already collected survive that denial and the result
|
|
1494
|
+
// is marked incomplete rather than being published as a complete tab.
|
|
1495
|
+
async function fetchGraphqlList(kind, mapRow, {
|
|
1496
|
+
signal,
|
|
1497
|
+
governor = null,
|
|
1498
|
+
previousRaw = null,
|
|
1499
|
+
// Seams, so the paging loop can be exercised without a live governor: the
|
|
1500
|
+
// settlement figure it produces was wrong for months of repositories with
|
|
1501
|
+
// more than one page, and nothing could see it.
|
|
1502
|
+
fetchPage = fetchGraphqlPage,
|
|
1503
|
+
admit = runAdmittedOperation,
|
|
1504
|
+
} = {}) {
|
|
1505
|
+
const connection = GRAPHQL_QUERIES[kind].connection;
|
|
1506
|
+
const operation = `tab:${kind}`;
|
|
1507
|
+
const first = await fetchPage(kind, { signal, operation });
|
|
1508
|
+
if (!first.ok) {
|
|
1509
|
+
const error = first.failure ?? new Error(`GraphQL ${kind} page unavailable (${first.reason})`);
|
|
1510
|
+
error.budgetObservations = first.observations;
|
|
1511
|
+
throw error;
|
|
1512
|
+
}
|
|
1513
|
+
const rows = [];
|
|
1514
|
+
const observations = [...first.observations];
|
|
1515
|
+
// Only the first page. Pages past it opened their own reservation inside
|
|
1516
|
+
// runAdmittedOperation and settle against it, so adding them here charges the
|
|
1517
|
+
// same work twice -- and a settlement above its reservation is rejected as
|
|
1518
|
+
// corrupt, which silently leaks the tab's reservation and discards every
|
|
1519
|
+
// budget observation the fetch gathered.
|
|
1520
|
+
const envelopeSpent = first.observedCost;
|
|
1521
|
+
let spent = first.observedCost;
|
|
1522
|
+
let page = graphqlConnection(first, connection);
|
|
1523
|
+
let incomplete = false;
|
|
1524
|
+
let overrun = first.overrun;
|
|
1525
|
+
let totalCount = page?.totalCount ?? null;
|
|
1526
|
+
for (const node of page?.nodes ?? []) rows.push(mapRow(node));
|
|
1527
|
+
while (page?.pageInfo?.hasNextPage && rows.length < LIST_LIMIT) {
|
|
1528
|
+
const cursor = page.pageInfo.endCursor;
|
|
1529
|
+
if (typeof cursor !== "string" || cursor.length === 0) { incomplete = true; break; }
|
|
1530
|
+
if (!governor) { incomplete = true; break; }
|
|
1531
|
+
const admitted = await admit({
|
|
1532
|
+
...governor,
|
|
1533
|
+
operation: `page:${kind}`,
|
|
1534
|
+
priority: "background",
|
|
1535
|
+
signal,
|
|
1536
|
+
run: (admittedSignal) => fetchPage(kind, { signal: admittedSignal, after: cursor, operation: `page:${kind}` }),
|
|
1537
|
+
});
|
|
1538
|
+
// A later page that is denied, fails, or returns errors leaves the rows
|
|
1539
|
+
// already gathered exactly as they are. Losing page one because page two
|
|
1540
|
+
// was refused would turn a budget decision into data loss.
|
|
1541
|
+
//
|
|
1542
|
+
// Note that runAdmittedOperation settles these at their *declared* cost, not
|
|
1543
|
+
// their observed one -- conservative, so never an under-charge. A later
|
|
1544
|
+
// page's real meter reaches the ledger through the observations forwarded to
|
|
1545
|
+
// the tab settlement rather than through its own reservation.
|
|
1546
|
+
// Evidence first, and unconditionally: a page that failed as data still
|
|
1547
|
+
// observed the meter, and dropping that is the one thing this phase says
|
|
1548
|
+
// it will not do.
|
|
1549
|
+
if (admitted.value?.observations) observations.push(...admitted.value.observations);
|
|
1550
|
+
if (admitted.value?.overrun) overrun = true;
|
|
1551
|
+
if (!admitted.ok || !admitted.value?.ok) { incomplete = true; break; }
|
|
1552
|
+
const next = admitted.value;
|
|
1553
|
+
spent += next.observedCost;
|
|
1554
|
+
page = graphqlConnection(next, connection);
|
|
1555
|
+
if (!page) { incomplete = true; break; }
|
|
1556
|
+
for (const node of page.nodes) rows.push(mapRow(node));
|
|
1557
|
+
}
|
|
1558
|
+
if (page?.pageInfo?.hasNextPage && rows.length >= LIST_LIMIT) incomplete = true;
|
|
1559
|
+
if (overrun) {
|
|
1560
|
+
// Recorded at its real cost above; suspended here until the resource resets,
|
|
1561
|
+
// which is the soonest a corrected bound could be trusted again.
|
|
1562
|
+
const reset = observations.at(-1)?.resetMs;
|
|
1563
|
+
pauseOperation(operation, Number.isFinite(reset) ? reset + BUDGET_RESET_GRACE_MS : Date.now() + BUDGET_PROBE_MS);
|
|
1564
|
+
}
|
|
1565
|
+
const limited = rows.slice(0, LIST_LIMIT);
|
|
1566
|
+
// Serialized rows are the payload identity the unchanged-frame suppression
|
|
1567
|
+
// and the cache both compare on, so it must cover exactly what is rendered.
|
|
1568
|
+
const raw = JSON.stringify({ v: GRAPHQL_QUERY_VERSION, totalCount, incomplete, rows: limited });
|
|
1170
1569
|
return {
|
|
1171
|
-
raw,
|
|
1172
|
-
limit
|
|
1173
|
-
//
|
|
1174
|
-
//
|
|
1175
|
-
|
|
1176
|
-
|
|
1177
|
-
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
})),
|
|
1570
|
+
raw: raw === previousRaw ? previousRaw : raw,
|
|
1571
|
+
// An incomplete walk reports the rows it has as its own limit, so the
|
|
1572
|
+
// existing truncation indicator fires. "More rows exist than are shown" is
|
|
1573
|
+
// exactly what incomplete means, and publishing 50 of 150 rows as a
|
|
1574
|
+
// complete tab is indistinguishable from a repository with 50 open issues.
|
|
1575
|
+
limit: incomplete ? limited.length : LIST_LIMIT,
|
|
1576
|
+
restSpent: 0,
|
|
1577
|
+
graphqlSpent: envelopeSpent,
|
|
1578
|
+
// What the whole walk cost, for reporting. Never the settlement figure.
|
|
1579
|
+
graphqlSpentTotal: spent,
|
|
1580
|
+
observations,
|
|
1581
|
+
incomplete,
|
|
1582
|
+
parse: () => limited,
|
|
1185
1583
|
};
|
|
1186
1584
|
}
|
|
1187
1585
|
|
|
1188
|
-
function
|
|
1189
|
-
return
|
|
1190
|
-
"pr",
|
|
1191
|
-
"list",
|
|
1192
|
-
...repoArgs(),
|
|
1193
|
-
"--state",
|
|
1194
|
-
"open",
|
|
1195
|
-
"--limit",
|
|
1196
|
-
String(LIST_LIMIT),
|
|
1197
|
-
...SORT_RECENT,
|
|
1198
|
-
"--json",
|
|
1199
|
-
"number,title,author,headRefName,isDraft,reviewDecision,createdAt,updatedAt",
|
|
1200
|
-
];
|
|
1586
|
+
async function fetchIssues(signal, { governor = null, previousRaw = null } = {}) {
|
|
1587
|
+
return fetchGraphqlList("issues", ISSUE_ROW, { signal, governor, previousRaw });
|
|
1201
1588
|
}
|
|
1202
1589
|
|
|
1203
|
-
async function fetchPRs(signal) {
|
|
1204
|
-
|
|
1205
|
-
return {
|
|
1206
|
-
raw,
|
|
1207
|
-
limit: LIST_LIMIT,
|
|
1208
|
-
// Zero for the same reason as issues: --search makes this a GraphQL call.
|
|
1209
|
-
restSpent: REST_PER_FETCH.prs,
|
|
1210
|
-
graphqlSpent: GRAPHQL_PER_FETCH.prs,
|
|
1211
|
-
parse: () =>
|
|
1212
|
-
parseJsonOutput(raw).map((p) => ({
|
|
1213
|
-
number: p.number,
|
|
1214
|
-
title: safe(p.title),
|
|
1215
|
-
author: safe(p.author?.login ?? ""),
|
|
1216
|
-
headRefName: safe(p.headRefName),
|
|
1217
|
-
isDraft: p.isDraft,
|
|
1218
|
-
reviewDecision: p.reviewDecision,
|
|
1219
|
-
updatedAt: p.updatedAt,
|
|
1220
|
-
})),
|
|
1221
|
-
};
|
|
1590
|
+
async function fetchPRs(signal, { governor = null, previousRaw = null } = {}) {
|
|
1591
|
+
return fetchGraphqlList("prs", PR_ROW, { signal, governor, previousRaw });
|
|
1222
1592
|
}
|
|
1223
1593
|
|
|
1224
1594
|
// The three alert endpoints were three near-identical 26-line blocks that had
|
|
@@ -1320,11 +1690,14 @@ const OPERATION_COSTS = Object.freeze({
|
|
|
1320
1690
|
"tab:actions-runs": { core: 1, graphql: 0 },
|
|
1321
1691
|
"tab:actions-workflows": { core: 1, graphql: 0 },
|
|
1322
1692
|
"tab:security-endpoint": { core: 1, graphql: 0 },
|
|
1323
|
-
"failure-context:repository": { core: 0, graphql:
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
"
|
|
1327
|
-
"
|
|
1693
|
+
"failure-context:repository": { core: 0, graphql: GRAPHQL_OBSERVER_POINTS },
|
|
1694
|
+
// Each page past the first reserves its own envelope, so a denial there is a
|
|
1695
|
+
// scheduling outcome rather than an unadmitted overspend.
|
|
1696
|
+
"page:issues": { core: 0, graphql: GRAPHQL_PAGE_POINTS },
|
|
1697
|
+
"page:prs": { core: 0, graphql: GRAPHQL_PAGE_POINTS },
|
|
1698
|
+
// The one claimed GraphQL observer. Not free, and not describable as free:
|
|
1699
|
+
// it selects the meter and pays a point for it.
|
|
1700
|
+
"graphql-observer": { core: 0, graphql: GRAPHQL_OBSERVER_POINTS },
|
|
1328
1701
|
"doctor:repository": { core: 0, graphql: 1 },
|
|
1329
1702
|
"doctor:actions-runs": { core: 1, graphql: 0 },
|
|
1330
1703
|
"doctor:actions-workflows": { core: 1, graphql: 0 },
|
|
@@ -1337,7 +1710,6 @@ const OPERATION_COSTS = Object.freeze({
|
|
|
1337
1710
|
"budget-core-observer": { core: 1, graphql: 0 },
|
|
1338
1711
|
"rate-limit": { core: 0, graphql: 0 },
|
|
1339
1712
|
"version": { core: 0, graphql: 0 },
|
|
1340
|
-
"auth-status": { core: 0, graphql: 0 },
|
|
1341
1713
|
"local-git": { core: 0, graphql: 0 },
|
|
1342
1714
|
});
|
|
1343
1715
|
|
|
@@ -1726,6 +2098,7 @@ function scheduleIntents({
|
|
|
1726
2098
|
cursors = {},
|
|
1727
2099
|
nowMs,
|
|
1728
2100
|
maxGrants = Number.POSITIVE_INFINITY,
|
|
2101
|
+
manualStreak = 0,
|
|
1729
2102
|
}) {
|
|
1730
2103
|
const valid = [];
|
|
1731
2104
|
const prunedIntentIds = [];
|
|
@@ -1766,11 +2139,29 @@ function scheduleIntents({
|
|
|
1766
2139
|
const liveLeaseCount = countLiveLeases(leases, nowMs);
|
|
1767
2140
|
const priorities = [...new Set(valid.map((intent) => intent.normalizedPriority))].sort((a, b) => a - b);
|
|
1768
2141
|
|
|
1769
|
-
|
|
1770
|
-
|
|
2142
|
+
let streak = Number.isSafeInteger(manualStreak) && manualStreak > 0 ? manualStreak : 0;
|
|
2143
|
+
const grantedIds = new Set();
|
|
2144
|
+
// Priority still decides every other question. This says only that "higher
|
|
2145
|
+
// priority" cannot mean "always": once manual work has taken the last three
|
|
2146
|
+
// turns and an active owner is waiting, that owner takes this one. The turn
|
|
2147
|
+
// is a single grant, and it is subject to exactly the same budget, phase and
|
|
2148
|
+
// lane checks as any other -- fairness reorders work, it never admits work
|
|
2149
|
+
// that could not otherwise be paid for.
|
|
2150
|
+
const owedActiveTurn = streak >= MANUAL_GRANT_STREAK_LIMIT &&
|
|
2151
|
+
valid.some((intent) => intent.normalizedPriority === REQUEST_PRIORITIES.active);
|
|
2152
|
+
const passes = owedActiveTurn
|
|
2153
|
+
? [{ priority: REQUEST_PRIORITIES.active, limit: 1, record: false },
|
|
2154
|
+
...priorities.map((priority) => ({ priority, limit: Number.POSITIVE_INFINITY, record: true }))]
|
|
2155
|
+
: priorities.map((priority) => ({ priority, limit: Number.POSITIVE_INFINITY, record: true }));
|
|
2156
|
+
|
|
2157
|
+
for (const pass of passes) {
|
|
2158
|
+
const priority = pass.priority;
|
|
2159
|
+
const pending = valid.filter((intent) =>
|
|
2160
|
+
intent.normalizedPriority === priority && !grantedIds.has(intent.id));
|
|
1771
2161
|
const roundRobinState = createRoundRobinState(pending, updatedCursors);
|
|
2162
|
+
let grantedThisPass = 0;
|
|
1772
2163
|
while (pending.length > 0) {
|
|
1773
|
-
if (grants.length >= maxGrants) break;
|
|
2164
|
+
if (grants.length >= maxGrants || grantedThisPass >= pass.limit) break;
|
|
1774
2165
|
const intent = pending.splice(nextRoundRobinIndex(pending, roundRobinState), 1)[0];
|
|
1775
2166
|
const resources = RATE_RESOURCES.filter((resource) => intent.costs[resource] > 0);
|
|
1776
2167
|
const decisions = Object.fromEntries(resources.map((resource) => [
|
|
@@ -1786,7 +2177,10 @@ function scheduleIntents({
|
|
|
1786
2177
|
]));
|
|
1787
2178
|
const blocked = resources.find((resource) => decisions[resource].mode !== "open");
|
|
1788
2179
|
if (blocked) {
|
|
1789
|
-
|
|
2180
|
+
// Only the ordinary pass records a denial. The owed-turn pass looks at
|
|
2181
|
+
// the same intents again immediately afterwards, and recording here
|
|
2182
|
+
// would report every one of them twice.
|
|
2183
|
+
if (pass.record) denied.push({ intentId: intent.id, resource: blocked, ...decisions[blocked] });
|
|
1790
2184
|
continue;
|
|
1791
2185
|
}
|
|
1792
2186
|
|
|
@@ -1801,6 +2195,7 @@ function scheduleIntents({
|
|
|
1801
2195
|
);
|
|
1802
2196
|
const expiring = resources.find((resource) => notBefore >= decisions[resource].resetMs);
|
|
1803
2197
|
if (expiring) {
|
|
2198
|
+
if (!pass.record) continue;
|
|
1804
2199
|
denied.push({
|
|
1805
2200
|
intentId: intent.id,
|
|
1806
2201
|
resource: expiring,
|
|
@@ -1835,6 +2230,12 @@ function scheduleIntents({
|
|
|
1835
2230
|
reservation.sharingOwnerLeaseIds = sharingOwnerLeaseIds;
|
|
1836
2231
|
}
|
|
1837
2232
|
grants.push(reservation);
|
|
2233
|
+
grantedIds.add(intent.id);
|
|
2234
|
+
grantedThisPass += 1;
|
|
2235
|
+
// A manual grant extends the run; anything else ends it. The counter has
|
|
2236
|
+
// to persist, because the run this bounds is one manual intent per
|
|
2237
|
+
// planning pass rather than several within one.
|
|
2238
|
+
streak = priority === REQUEST_PRIORITIES.manual ? streak + 1 : 0;
|
|
1838
2239
|
for (const resource of resources) {
|
|
1839
2240
|
chargedTotals[resource] += intent.costs[resource];
|
|
1840
2241
|
updatedLanes[resource] = {
|
|
@@ -1853,16 +2254,853 @@ function scheduleIntents({
|
|
|
1853
2254
|
cursors: updatedCursors,
|
|
1854
2255
|
denied,
|
|
1855
2256
|
prunedIntentIds,
|
|
2257
|
+
manualStreak: streak,
|
|
1856
2258
|
};
|
|
1857
2259
|
}
|
|
1858
2260
|
|
|
2261
|
+
// ---------- Credential identities and restart-safe coordination ----------
|
|
2262
|
+
const IDENTITY_VERSION = 1;
|
|
2263
|
+
const IDENTITY_ATTEMPT_WINDOW_MS = 15 * 60_000;
|
|
2264
|
+
const IDENTITY_ATTEMPT_LIMIT = 3;
|
|
2265
|
+
const IDENTITY_HOST_ATTEMPT_LIMIT = 12;
|
|
2266
|
+
const IDENTITY_MAX_ATTEMPTS = 4096;
|
|
2267
|
+
const HTTP_START_GAP_MS = 250;
|
|
2268
|
+
const HTTP_MAX_WAITERS = 128;
|
|
2269
|
+
// A permit is reclaimable once it cannot still belong to a live request. The
|
|
2270
|
+
// owner-death check alone cannot see a process that is stopped, or one whose
|
|
2271
|
+
// release lost its lock, so a permit outliving the subprocess timeout it guards
|
|
2272
|
+
// is stale by construction rather than by guess.
|
|
2273
|
+
const HTTP_PERMIT_MAX_MS = GH_TIMEOUT_MS + 5_000;
|
|
2274
|
+
// The longest a primary window can run. An attempt that never obtained reset
|
|
2275
|
+
// evidence is still covered once a full window has demonstrably elapsed since it
|
|
2276
|
+
// started, which is what keeps an unprovable charge from being retained forever.
|
|
2277
|
+
const IDENTITY_UNCERTAIN_MAX_MS = 3_600_000 + BUDGET_RESET_GRACE_MS;
|
|
2278
|
+
// Bound the in-band wait for the registry lock when a request is already
|
|
2279
|
+
// finishing. The permit is what other panes queue behind, so a slow release must
|
|
2280
|
+
// hand off to deferCompletion quickly instead of holding the whole machine.
|
|
2281
|
+
const IDENTITY_RELEASE_WAIT_MS = 500;
|
|
2282
|
+
// How long startup will wait for a warm local identity before painting. Long
|
|
2283
|
+
// enough to win the ordinary case, short enough that a stuck credential helper
|
|
2284
|
+
// costs a cache-cold first frame rather than a blank terminal.
|
|
2285
|
+
const WARM_IDENTITY_WAIT_MS = 1_000;
|
|
2286
|
+
// "busy" and "unwritable" say the registry could not be read or written *right
|
|
2287
|
+
// now*. Every other reason is a statement about the identity itself. Three
|
|
2288
|
+
// callers each drew this line differently -- retryIdentityCompletion retried
|
|
2289
|
+
// both, refresh() treated them as proof the identity was gone, and permit
|
|
2290
|
+
// acquisition threw on them -- so a 250 ms lock wait could be reported as an
|
|
2291
|
+
// account change. One definition keeps them from drifting apart again.
|
|
2292
|
+
const RETRYABLE_COORDINATION_REASONS = ["busy", "unwritable"];
|
|
2293
|
+
// 60 s doubling per prior attempt in the window. The two callers deliberately
|
|
2294
|
+
// count different prior sets -- bootstrap also has a per-host unknown-principal
|
|
2295
|
+
// allowance and the transport start gap to respect, while a known-capacity
|
|
2296
|
+
// observer is admitted normally and only the exceptional branch is throttled --
|
|
2297
|
+
// but the delay they apply is one law and belongs in one place.
|
|
2298
|
+
function attemptRetryDelayMs(priorAttempts) {
|
|
2299
|
+
return 60_000 * 2 ** priorAttempts;
|
|
2300
|
+
}
|
|
2301
|
+
|
|
2302
|
+
function retryableCoordination(reason) {
|
|
2303
|
+
return RETRYABLE_COORDINATION_REASONS.includes(reason);
|
|
2304
|
+
}
|
|
2305
|
+
|
|
2306
|
+
// A dead owner releases its permit immediately; a live owner keeps it only for
|
|
2307
|
+
// as long as a request could still be running under it. Without the second
|
|
2308
|
+
// test a stopped process, or one whose release never won the lock, holds the
|
|
2309
|
+
// single machine-wide permit forever and every pane queues behind it.
|
|
2310
|
+
function staleHttpPermit(permit, now, kill = process.kill.bind(process)) {
|
|
2311
|
+
if (pidIsDead(permit.pid, kill)) return true;
|
|
2312
|
+
return Number.isFinite(permit.startedAt) && now - permit.startedAt > HTTP_PERMIT_MAX_MS;
|
|
2313
|
+
}
|
|
2314
|
+
let runtimeIdentityCoordinator = null;
|
|
2315
|
+
|
|
2316
|
+
function privateIdentityDigest(...parts) {
|
|
2317
|
+
return createHash("sha256").update(JSON.stringify(parts)).digest("hex");
|
|
2318
|
+
}
|
|
2319
|
+
|
|
2320
|
+
function identityRegistryRoot(pathOptions = {}) {
|
|
2321
|
+
return join(dirname(widthPreferencesPath(pathOptions)), "coordination-v2");
|
|
2322
|
+
}
|
|
2323
|
+
|
|
2324
|
+
function credentialEnvironmentNames(host) {
|
|
2325
|
+
return host === "github.com" || host.endsWith(".ghe.com")
|
|
2326
|
+
? ["GH_TOKEN", "GITHUB_TOKEN"] : ["GH_ENTERPRISE_TOKEN", "GITHUB_ENTERPRISE_TOKEN"];
|
|
2327
|
+
}
|
|
2328
|
+
|
|
2329
|
+
// Deliberately not memoized, despite being called on the render and request
|
|
2330
|
+
// paths. resolveEffectiveCredential brackets the `gh auth token` subprocess with
|
|
2331
|
+
// a before/after comparison of this value to catch a credential swapped while
|
|
2332
|
+
// resolution was in flight, so any cache whose window can cover that gap makes
|
|
2333
|
+
// the process bind the wrong credential -- and a local token can resolve in
|
|
2334
|
+
// under a millisecond. The cost is one statSync plus a digest of four numbers;
|
|
2335
|
+
// reduce the number of callers instead of caching the answer.
|
|
2336
|
+
function credentialConfigurationRevision(host, env = process.env) {
|
|
2337
|
+
const names = credentialEnvironmentNames(host);
|
|
2338
|
+
const selected = names.find((name) => env[name]);
|
|
2339
|
+
if (selected) return privateIdentityDigest(host, String(env[selected]));
|
|
2340
|
+
const directory = effectiveGhConfigDir({ env });
|
|
2341
|
+
let stamp = null;
|
|
2342
|
+
try {
|
|
2343
|
+
const value = statSync(join(directory, "hosts.yml"));
|
|
2344
|
+
stamp = [value.dev, value.ino, value.size, value.mtimeMs];
|
|
2345
|
+
} catch { /* missing local configuration is resolved honestly by gh */ }
|
|
2346
|
+
return privateIdentityDigest(host, directory, stamp);
|
|
2347
|
+
}
|
|
2348
|
+
|
|
2349
|
+
async function resolveEffectiveCredential({ host, env = process.env, runLocalToken } = {}) {
|
|
2350
|
+
host = normalizeHost(host);
|
|
2351
|
+
if (!host) return { ok: false, reason: "unknown-host" };
|
|
2352
|
+
const names = credentialEnvironmentNames(host);
|
|
2353
|
+
const initialRevision = credentialConfigurationRevision(host, env);
|
|
2354
|
+
let token;
|
|
2355
|
+
try {
|
|
2356
|
+
const selected = names.find((name) => env[name]);
|
|
2357
|
+
if (selected) token = String(env[selected]);
|
|
2358
|
+
else {
|
|
2359
|
+
// This seam never passes through runGh/logGh and never exposes the child
|
|
2360
|
+
// error, stdout or stderr. Credentials exist only in this lexical scope.
|
|
2361
|
+
const result = await (runLocalToken ?? (async (args) => execFileAsync("gh", args, {
|
|
2362
|
+
env: { ...env, ...GH_ENV_OVERRIDES }, timeout: GH_TIMEOUT_MS,
|
|
2363
|
+
maxBuffer: 1024 * 1024, killSignal: "SIGKILL",
|
|
2364
|
+
})))(["auth", "token", "--hostname", host]);
|
|
2365
|
+
token = String(typeof result === "string" ? result : result.stdout ?? "").trim();
|
|
2366
|
+
}
|
|
2367
|
+
if (credentialConfigurationRevision(host, env) !== initialRevision) return { ok: false, reason: "credential-changed" };
|
|
2368
|
+
if (!token || /\s/.test(token)) return { ok: false, reason: "credential-unavailable" };
|
|
2369
|
+
const credentialKey = privateIdentityDigest("credential-v1", host, token);
|
|
2370
|
+
token = null;
|
|
2371
|
+
return { ok: true, value: { host, credentialKey, configurationRevision: initialRevision } };
|
|
2372
|
+
} catch {
|
|
2373
|
+
return { ok: false, reason: "credential-unavailable" };
|
|
2374
|
+
}
|
|
2375
|
+
}
|
|
2376
|
+
|
|
2377
|
+
function emptyIdentityRegistry() {
|
|
2378
|
+
return { version: IDENTITY_VERSION, migration: { activated: false, holdUntil: 0 }, identities: {}, attempts: {}, hosts: {} };
|
|
2379
|
+
}
|
|
2380
|
+
|
|
2381
|
+
function normalizeIdentityRegistry(raw) {
|
|
2382
|
+
if (!exactKeys(raw, ["version", "migration", "identities", "attempts", "hosts"]) || raw.version !== IDENTITY_VERSION ||
|
|
2383
|
+
!exactKeys(raw.migration, ["activated", "holdUntil"]) || typeof raw.migration.activated !== "boolean" ||
|
|
2384
|
+
!Number.isFinite(raw.migration.holdUntil) || raw.migration.holdUntil < 0 ||
|
|
2385
|
+
!isRecord(raw.identities) || !isRecord(raw.attempts) || !isRecord(raw.hosts) ||
|
|
2386
|
+
Object.keys(raw.attempts).length > IDENTITY_MAX_ATTEMPTS) return null;
|
|
2387
|
+
const digest = (value) => typeof value === "string" && /^[0-9a-f]{64}$/.test(value);
|
|
2388
|
+
const time = (value) => Number.isFinite(value) && value >= 0;
|
|
2389
|
+
for (const [key, identity] of Object.entries(raw.identities)) {
|
|
2390
|
+
if (!digest(key) || !exactKeys(identity, ["host", "kind", "id", "login", "quotaKey", "accessKey", "generation", "observedAt"]) ||
|
|
2391
|
+
!normalizeHost(identity.host) || identity.kind !== "user" || !Number.isSafeInteger(identity.id) || identity.id <= 0 ||
|
|
2392
|
+
typeof identity.login !== "string" || identity.login !== safe(identity.login) || !digest(identity.quotaKey) || !digest(identity.accessKey) ||
|
|
2393
|
+
!Number.isSafeInteger(identity.generation) || identity.generation < 1 || !time(identity.observedAt)) return null;
|
|
2394
|
+
}
|
|
2395
|
+
for (const [id, attempt] of Object.entries(raw.attempts)) {
|
|
2396
|
+
if (!validGovernorId(id) || !exactKeys(attempt, ["credentialKey", "host", "resource", "startedAt", "retryAt", "ownerPid", "nonce", "status", "quotaKey", "resetMs", "accounted", "imported", "exceptional"]) ||
|
|
2397
|
+
!digest(attempt.credentialKey) || !normalizeHost(attempt.host) || !RATE_RESOURCES.includes(attempt.resource) ||
|
|
2398
|
+
!time(attempt.startedAt) || !time(attempt.retryAt) || !Number.isSafeInteger(attempt.ownerPid) || attempt.ownerPid < 1 || !validGovernorId(attempt.nonce) ||
|
|
2399
|
+
!["started", "finished", "uncertain"].includes(attempt.status) || (attempt.quotaKey !== null && !digest(attempt.quotaKey)) ||
|
|
2400
|
+
(attempt.resetMs !== null && !time(attempt.resetMs)) || typeof attempt.accounted !== "boolean" || typeof attempt.imported !== "boolean" || typeof attempt.exceptional !== "boolean") return null;
|
|
2401
|
+
}
|
|
2402
|
+
for (const [host, transport] of Object.entries(raw.hosts)) {
|
|
2403
|
+
if (!isRecord(transport)) return null;
|
|
2404
|
+
const keys = ["lastStartedAt", "cooldownUntil", "permit"];
|
|
2405
|
+
if (Object.hasOwn(transport, "waiters")) keys.push("waiters");
|
|
2406
|
+
if (Object.hasOwn(transport, "throttle")) keys.push("throttle");
|
|
2407
|
+
if (!normalizeHost(host) || !exactKeys(transport, keys) ||
|
|
2408
|
+
!time(transport.lastStartedAt) || !time(transport.cooldownUntil)) return null;
|
|
2409
|
+
if (transport.permit !== null && (!exactKeys(transport.permit, ["pid", "nonce", "startedAt"]) ||
|
|
2410
|
+
!Number.isSafeInteger(transport.permit.pid) || transport.permit.pid < 1 || !validGovernorId(transport.permit.nonce) || !time(transport.permit.startedAt))) return null;
|
|
2411
|
+
if (!Object.hasOwn(transport, "throttle")) transport.throttle = emptyTransportThrottle();
|
|
2412
|
+
if (!isRecord(transport.throttle) || !exactKeys(transport.throttle, ["attempts", "lastAt", "paused"]) ||
|
|
2413
|
+
!Number.isSafeInteger(transport.throttle.attempts) || transport.throttle.attempts < 0 ||
|
|
2414
|
+
transport.throttle.attempts > THROTTLE_PAUSE_AFTER ||
|
|
2415
|
+
!time(transport.throttle.lastAt) || typeof transport.throttle.paused !== "boolean") return null;
|
|
2416
|
+
if (!Object.hasOwn(transport, "waiters")) transport.waiters = [];
|
|
2417
|
+
if (!Array.isArray(transport.waiters) || transport.waiters.length > HTTP_MAX_WAITERS ||
|
|
2418
|
+
new Set(transport.waiters.map((waiter) => waiter?.nonce)).size !== transport.waiters.length ||
|
|
2419
|
+
transport.waiters.some((waiter) => !exactKeys(waiter, ["pid", "nonce", "queuedAt", "deadline"]) ||
|
|
2420
|
+
!Number.isSafeInteger(waiter.pid) || waiter.pid < 1 || !validGovernorId(waiter.nonce) ||
|
|
2421
|
+
!time(waiter.queuedAt) || !time(waiter.deadline) || waiter.deadline <= waiter.queuedAt ||
|
|
2422
|
+
waiter.deadline - waiter.queuedAt > GH_TIMEOUT_MS)) return null;
|
|
2423
|
+
}
|
|
2424
|
+
return raw;
|
|
2425
|
+
}
|
|
2426
|
+
|
|
2427
|
+
function withIdentityRegistry(root, operation, { now = Date.now(), kill = process.kill.bind(process) } = {}) {
|
|
2428
|
+
const path = join(root, "registry.json");
|
|
2429
|
+
const scope = { path, hash: "identity-registry", host: "github.com", identityProvider: null, kill };
|
|
2430
|
+
return withGovernorLock(scope, () => {
|
|
2431
|
+
// Read and parse are separated for the same reason readGovernorState
|
|
2432
|
+
// separates them: EACCES, EIO or EMFILE mean "try again", while unparseable
|
|
2433
|
+
// bytes mean "stop and preserve the evidence". Collapsing the two reported a
|
|
2434
|
+
// transient descriptor shortage as permanent corruption.
|
|
2435
|
+
let raw = null;
|
|
2436
|
+
try { raw = readFileSync(path, "utf8"); }
|
|
2437
|
+
catch (error) { if (error?.code !== "ENOENT") return { ok: false, reason: "unwritable" }; }
|
|
2438
|
+
let state;
|
|
2439
|
+
if (raw === null) state = emptyIdentityRegistry();
|
|
2440
|
+
else {
|
|
2441
|
+
try { state = normalizeIdentityRegistry(JSON.parse(raw)); }
|
|
2442
|
+
catch { return { ok: false, reason: "corrupt" }; }
|
|
2443
|
+
}
|
|
2444
|
+
if (!state) return { ok: false, reason: "corrupt" };
|
|
2445
|
+
// Captured before the prunes below so their deletions still count as a
|
|
2446
|
+
// change. Every governor operation -- including read-only inspection --
|
|
2447
|
+
// funnels through here, and each one used to serialize and rename the
|
|
2448
|
+
// registry while holding the one machine-wide lock.
|
|
2449
|
+
const before = JSON.stringify(state);
|
|
2450
|
+
for (const transport of Object.values(state.hosts)) {
|
|
2451
|
+
transport.waiters = transport.waiters.filter((waiter) => waiter.deadline > now && !pidIsDead(waiter.pid, kill));
|
|
2452
|
+
}
|
|
2453
|
+
for (const [id, attempt] of Object.entries(state.attempts)) {
|
|
2454
|
+
// Past its rolling window *and* past any backoff it still imposes. retryAt
|
|
2455
|
+
// is what carries known exhaustion (settlement pushes it out to reset plus
|
|
2456
|
+
// grace, beyond the window), so testing it is what lets an aged-out entry
|
|
2457
|
+
// be dropped without forgiving a hold that is still in force.
|
|
2458
|
+
if (attempt.startedAt + IDENTITY_ATTEMPT_WINDOW_MS > now || attempt.retryAt > now) continue;
|
|
2459
|
+
const identity = state.identities[attempt.credentialKey];
|
|
2460
|
+
// No identity for this credential means no quota scope, so there is no
|
|
2461
|
+
// ledger receipt the attempt could be holding and nothing to reconcile.
|
|
2462
|
+
// These are the failed bootstraps -- they never reach importIdentityDebts,
|
|
2463
|
+
// so they used to be immortal and wedged claimIdentityBootstrap at
|
|
2464
|
+
// IDENTITY_MAX_ATTEMPTS permanently, with deleting registry.json by hand
|
|
2465
|
+
// as the only way out.
|
|
2466
|
+
if (!identity) { delete state.attempts[id]; continue; }
|
|
2467
|
+
const scope = { ...createQuotaScope({ ...identity, credentialKey: attempt.credentialKey }, { root, now: () => now, kill }), rootLocked: true };
|
|
2468
|
+
const retired = withGovernorLock(scope, () => {
|
|
2469
|
+
const ledger = readIdentityQuotaState(state, scope, now);
|
|
2470
|
+
if (!ledger.ok || ledger.missing) return ledger.ok ? { ok: false, reason: "corrupt" } : ledger;
|
|
2471
|
+
const budget = ledger.value.budgets[attempt.resource];
|
|
2472
|
+
const resetCovered = attempt.resetMs !== null
|
|
2473
|
+
? now >= attempt.resetMs && budget?.resetMs > attempt.resetMs
|
|
2474
|
+
// An attempt whose observer never returned core evidence has no reset
|
|
2475
|
+
// of its own to wait for. A window boundary proven to fall after it
|
|
2476
|
+
// started covers its charge just as well, and is the only thing that
|
|
2477
|
+
// stops the entry from being retained for the life of the install.
|
|
2478
|
+
: budget?.resetMs > attempt.startedAt + IDENTITY_UNCERTAIN_MAX_MS;
|
|
2479
|
+
if (!attempt.accounted && !resetCovered) return { ok: true, value: false };
|
|
2480
|
+
delete ledger.value.reservations[`reservation:${id}`];
|
|
2481
|
+
const write = writeGovernorState(scope.path, ledger.value);
|
|
2482
|
+
return write.ok ? { ok: true, value: true } : write;
|
|
2483
|
+
});
|
|
2484
|
+
if (!retired.ok) return retired;
|
|
2485
|
+
if (retired.value) delete state.attempts[id];
|
|
2486
|
+
}
|
|
2487
|
+
const result = operation(state, now);
|
|
2488
|
+
if (!normalizeIdentityRegistry(state)) return { ok: false, reason: "corrupt" };
|
|
2489
|
+
if (JSON.stringify(state) === before) return result;
|
|
2490
|
+
const written = writeGovernorState(path, state);
|
|
2491
|
+
return written.ok ? result : written;
|
|
2492
|
+
});
|
|
2493
|
+
}
|
|
2494
|
+
|
|
2495
|
+
function inspectIdentityRegistry(root, options) {
|
|
2496
|
+
return withIdentityRegistry(root, (state) => ({ ok: true, value: structuredClone(state) }), options);
|
|
2497
|
+
}
|
|
2498
|
+
|
|
2499
|
+
function inspectLegacyMigration(root, state, now) {
|
|
2500
|
+
let names;
|
|
2501
|
+
try { names = readdirSync(dirname(root)).filter((name) => /^rate-governor-v1-[0-9a-f]{64}\.json$/.test(name)); }
|
|
2502
|
+
catch (error) { return { ok: false, reason: error.code === "ENOENT" ? "unwritable" : "corrupt" }; }
|
|
2503
|
+
let holdUntil = state.migration.holdUntil;
|
|
2504
|
+
for (const name of names) {
|
|
2505
|
+
let raw;
|
|
2506
|
+
try { raw = JSON.parse(readFileSync(join(dirname(root), name), "utf8")); } catch { return { ok: false, reason: "legacy-corrupt" }; }
|
|
2507
|
+
// Evidence is never rewritten. Recognize only the pinned legacy schemas.
|
|
2508
|
+
// Every protocol an older pane could still be running, newest first. The set
|
|
2509
|
+
// grows as GOVERNOR_STATE_VERSION advances: a version bump makes an old
|
|
2510
|
+
// binary fail closed on its own gate, but this side must still recognise
|
|
2511
|
+
// that binary's file well enough to see it holds a live lease.
|
|
2512
|
+
const legacy = LEGACY_GOVERNOR_VERSIONS
|
|
2513
|
+
.map((version) => readLegacyGovernorState(raw, now, version))
|
|
2514
|
+
.find(Boolean) ?? migrateGovernorState(raw, now);
|
|
2515
|
+
if (legacy && raw.version === 1 && raw.budgets?.core) {
|
|
2516
|
+
const core = raw.budgets.core;
|
|
2517
|
+
const preserved = normalizeGovernorBudget({ ...core, source: "core-observer", factorBaseline: { epoch: core.epoch, used: core.used, observedAt: core.observedAt }, knownLocalUsed: 0 }, now, "core");
|
|
2518
|
+
if (!preserved) return { ok: false, reason: "legacy-corrupt" };
|
|
2519
|
+
// This is a migration hold view only, never published as new authority.
|
|
2520
|
+
legacy.budgets.core = preserved;
|
|
2521
|
+
legacy.epochs.core = preserved.epoch;
|
|
2522
|
+
}
|
|
2523
|
+
if (!legacy) return { ok: false, reason: "legacy-corrupt" };
|
|
2524
|
+
if (Object.values(legacy.leases).some((lease) => lease.expiresAt > now)) return { ok: false, reason: "restart-required" };
|
|
2525
|
+
for (const resource of RATE_RESOURCES) {
|
|
2526
|
+
const budget = legacy.budgets[resource];
|
|
2527
|
+
holdUntil = Math.max(holdUntil, budget?.blockUntil ?? 0);
|
|
2528
|
+
const uncertain = Object.values(legacy.reservations).some((reservation) =>
|
|
2529
|
+
["started", "completed"].includes(reservation.status) && reservationCost(reservation, resource, legacy.leases, now) > 0);
|
|
2530
|
+
if (uncertain) {
|
|
2531
|
+
const reset = budget?.resetMs;
|
|
2532
|
+
if (!Number.isFinite(reset)) return { ok: false, reason: "legacy-unresolved" };
|
|
2533
|
+
holdUntil = Math.max(holdUntil, reset + BUDGET_RESET_GRACE_MS);
|
|
2534
|
+
}
|
|
2535
|
+
}
|
|
2536
|
+
}
|
|
2537
|
+
state.migration.holdUntil = holdUntil;
|
|
2538
|
+
if (holdUntil > now) return { ok: false, reason: "migration-hold", retryAt: holdUntil };
|
|
2539
|
+
// Deliberately a record, not a gate. Re-listing the legacy directory on every
|
|
2540
|
+
// operation is what satisfies "detect reappearing legacy leases during
|
|
2541
|
+
// operation": an old binary started after migration must still be able to
|
|
2542
|
+
// pause new admission, and it cannot announce itself through a sentinel it
|
|
2543
|
+
// does not read. Short-circuiting on this flag would be faster and wrong.
|
|
2544
|
+
state.migration.activated = true;
|
|
2545
|
+
return { ok: true };
|
|
2546
|
+
}
|
|
2547
|
+
|
|
2548
|
+
function claimIdentityBootstrap(root, { credentialKey, host, resource = "core", now = Date.now(), kill = process.kill.bind(process) } = {}) {
|
|
2549
|
+
return withIdentityRegistry(root, (state) => {
|
|
2550
|
+
const migration = inspectLegacyMigration(root, state, now);
|
|
2551
|
+
if (!migration.ok) return migration;
|
|
2552
|
+
const transport = state.hosts[host] ??= { lastStartedAt: 0, cooldownUntil: 0, permit: null };
|
|
2553
|
+
if (transport.permit && staleHttpPermit(transport.permit, now, kill)) transport.permit = null;
|
|
2554
|
+
if (transport.permit || transport.waiters?.length) return { ok: false, reason: "identity-busy", retryAt: now + 1000 };
|
|
2555
|
+
const attempts = Object.values(state.attempts);
|
|
2556
|
+
const recent = attempts.filter((attempt) => attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
|
|
2557
|
+
const own = recent.filter((attempt) => attempt.credentialKey === credentialKey && attempt.resource === resource && attempt.exceptional);
|
|
2558
|
+
const unknown = recent.filter((attempt) => attempt.host === host && attempt.exceptional);
|
|
2559
|
+
const retryAt = Math.max(transport.cooldownUntil, transport.lastStartedAt + HTTP_START_GAP_MS,
|
|
2560
|
+
...attempts.filter((attempt) => attempt.credentialKey === credentialKey && attempt.resource === resource).map((attempt) => attempt.retryAt),
|
|
2561
|
+
own.length >= IDENTITY_ATTEMPT_LIMIT ? Math.min(...own.map((attempt) => attempt.startedAt)) + IDENTITY_ATTEMPT_WINDOW_MS : 0,
|
|
2562
|
+
unknown.length >= IDENTITY_HOST_ATTEMPT_LIMIT ? Math.min(...unknown.map((attempt) => attempt.startedAt)) + IDENTITY_ATTEMPT_WINDOW_MS : 0);
|
|
2563
|
+
if (retryAt > now) return { ok: false, reason: "identity-backoff", retryAt };
|
|
2564
|
+
if (attempts.length >= IDENTITY_MAX_ATTEMPTS) return { ok: false, reason: "identity-capacity" };
|
|
2565
|
+
const id = governorId();
|
|
2566
|
+
const nonce = governorId();
|
|
2567
|
+
state.attempts[id] = { credentialKey, host, resource, startedAt: now, retryAt: now + attemptRetryDelayMs(own.length),
|
|
2568
|
+
ownerPid: process.pid, nonce, status: "started", quotaKey: null, resetMs: null, accounted: false, imported: false, exceptional: true };
|
|
2569
|
+
transport.permit = { pid: process.pid, nonce, startedAt: now };
|
|
2570
|
+
transport.lastStartedAt = now;
|
|
2571
|
+
return { ok: true, value: { id, nonce } };
|
|
2572
|
+
}, { now, kill });
|
|
2573
|
+
}
|
|
2574
|
+
|
|
2575
|
+
function readIdentityQuotaState(registry, scope, now) {
|
|
2576
|
+
const loaded = readGovernorState(scope.path, now);
|
|
2577
|
+
const known = Object.values(registry.identities).some((identity) => identity.quotaKey === scope.quotaKey) ||
|
|
2578
|
+
Object.values(registry.attempts).some((attempt) => attempt.imported && attempt.quotaKey === scope.quotaKey);
|
|
2579
|
+
return loaded.ok && loaded.missing && known ? { ok: false, reason: "corrupt" } : loaded;
|
|
2580
|
+
}
|
|
2581
|
+
|
|
2582
|
+
function importIdentityDebts(root, state, credentialKey, identity, now) {
|
|
2583
|
+
const scope = { ...createQuotaScope({ ...identity, credentialKey }, { root, now: () => now }), rootLocked: true };
|
|
2584
|
+
return withGovernorLock(scope, () => {
|
|
2585
|
+
const loaded = readIdentityQuotaState(state, scope, now);
|
|
2586
|
+
if (!loaded.ok) return loaded;
|
|
2587
|
+
const transferred = [];
|
|
2588
|
+
for (const [id, attempt] of Object.entries(state.attempts)) {
|
|
2589
|
+
if (attempt.credentialKey !== credentialKey || attempt.imported) continue;
|
|
2590
|
+
const reservationId = `reservation:${id}`;
|
|
2591
|
+
loaded.value.reservations[reservationId] ??= {
|
|
2592
|
+
leaseId: id, intentId: id, costs: { core: 0, graphql: 0, [attempt.resource]: 1 }, actualCosts: null,
|
|
2593
|
+
accountedCosts: { core: 0, graphql: 0, [attempt.resource]: attempt.accounted ? 1 : 0 },
|
|
2594
|
+
notBefore: attempt.startedAt, status: "started", epochs: { core: null, graphql: null },
|
|
2595
|
+
startedAt: attempt.startedAt, completedAt: null, outcome: null,
|
|
2596
|
+
};
|
|
2597
|
+
transferred.push(id);
|
|
2598
|
+
}
|
|
2599
|
+
// importIdentityDebts writes this ledger before the registry transaction
|
|
2600
|
+
// that records the mapping commits. A crash in between leaves a charged
|
|
2601
|
+
// receipt here whose attempt is gone from the registry, and nothing else
|
|
2602
|
+
// ever looks at it again -- it presents only as a permanently smaller
|
|
2603
|
+
// budget. Re-bootstrapping the same credential is exactly when it can be
|
|
2604
|
+
// recognised, and a full window since it started proves a reset covered it.
|
|
2605
|
+
for (const key of Object.keys(loaded.value.reservations)) {
|
|
2606
|
+
const attemptId = /^reservation:(.+)$/.exec(key)?.[1];
|
|
2607
|
+
if (!attemptId || state.attempts[attemptId]) continue;
|
|
2608
|
+
const reservation = loaded.value.reservations[key];
|
|
2609
|
+
if (reservation.startedAt + IDENTITY_UNCERTAIN_MAX_MS > now) continue;
|
|
2610
|
+
delete loaded.value.reservations[key];
|
|
2611
|
+
}
|
|
2612
|
+
const written = writeGovernorState(scope.path, loaded.value);
|
|
2613
|
+
if (!written.ok) return written;
|
|
2614
|
+
// Marked transferred only once the receipts are durable. withIdentityRegistry
|
|
2615
|
+
// persists the registry whatever this returns, so setting these before the
|
|
2616
|
+
// ledger write meant a failed write could leave the registry claiming a debt
|
|
2617
|
+
// had moved to a ledger that never received it -- and the `imported` guard
|
|
2618
|
+
// would then skip those attempts forever, dropping the conservative charge.
|
|
2619
|
+
for (const id of transferred) {
|
|
2620
|
+
state.attempts[id].quotaKey = identity.quotaKey;
|
|
2621
|
+
state.attempts[id].imported = true;
|
|
2622
|
+
}
|
|
2623
|
+
return written;
|
|
2624
|
+
});
|
|
2625
|
+
}
|
|
2626
|
+
|
|
2627
|
+
function finishIdentityBootstrap(root, { credentialKey, id, nonce, response = null, now = Date.now(), isCurrent = () => true } = {}) {
|
|
2628
|
+
return withIdentityRegistry(root, (state) => {
|
|
2629
|
+
const attempt = state.attempts[id];
|
|
2630
|
+
if (!attempt || attempt.nonce !== nonce || attempt.credentialKey !== credentialKey) return { ok: false, reason: "stale" };
|
|
2631
|
+
const transport = state.hosts[attempt.host];
|
|
2632
|
+
if (transport.permit?.nonce === nonce) transport.permit = null;
|
|
2633
|
+
attempt.status = response ? "finished" : "uncertain";
|
|
2634
|
+
const parsed = response?.body;
|
|
2635
|
+
const evidence = response?.rateLimit;
|
|
2636
|
+
applyTransportCooldown(transport, { retryAfter: response?.retryAfter, status: response?.status, secondary: response?.secondary, at: now });
|
|
2637
|
+
if (evidence?.resource === "core" && normalizeBudgetResource(evidence, now)) {
|
|
2638
|
+
attempt.resetMs = evidence.resetMs;
|
|
2639
|
+
if (evidence.remaining === 0) attempt.retryAt = Math.max(attempt.retryAt, evidence.resetMs + BUDGET_RESET_GRACE_MS);
|
|
2640
|
+
}
|
|
2641
|
+
if (!isCurrent()) return { ok: false, reason: "stale" };
|
|
2642
|
+
if ((response?.status ?? 200) !== 200 || !isRecord(parsed) || !Number.isSafeInteger(parsed.id) || parsed.id <= 0 || typeof parsed.login !== "string" ||
|
|
2643
|
+
parsed.login.length === 0 || !evidence || evidence.resource !== "core" || !normalizeBudgetResource(evidence, now)) return { ok: false, reason: "identity-unavailable" };
|
|
2644
|
+
const generation = state.identities[credentialKey]?.generation ?? 1;
|
|
2645
|
+
const identity = { host: attempt.host, kind: "user", id: parsed.id, login: safe(parsed.login),
|
|
2646
|
+
quotaKey: privateIdentityDigest("quota-v2", attempt.host, "user", parsed.id),
|
|
2647
|
+
accessKey: privateIdentityDigest("access-v2", credentialKey, generation), generation, observedAt: now };
|
|
2648
|
+
attempt.accounted = true;
|
|
2649
|
+
for (const previous of Object.values(state.attempts)) {
|
|
2650
|
+
if (previous.credentialKey === credentialKey && previous.resource === "core") previous.resetMs ??= evidence.resetMs;
|
|
2651
|
+
}
|
|
2652
|
+
// Record the mapping only after the ledger contains every debt. A crash
|
|
2653
|
+
// between writes replays stable reservation IDs while the root lock prevents
|
|
2654
|
+
// observers from deleting receipts before the mapping transaction finishes.
|
|
2655
|
+
const imported = importIdentityDebts(root, state, credentialKey, identity, now);
|
|
2656
|
+
if (!imported.ok) return imported;
|
|
2657
|
+
// The admitted identity proof supplies core authority. Retain its validator
|
|
2658
|
+
// so later observers can confirm unchanged identity without primary spend.
|
|
2659
|
+
{
|
|
2660
|
+
const scope = { ...createQuotaScope({ ...identity, credentialKey }, { root, now: () => now }), rootLocked: true };
|
|
2661
|
+
const seeded = withGovernorLock(scope, () => {
|
|
2662
|
+
const ledger = readIdentityQuotaState(state, scope, now);
|
|
2663
|
+
if (!ledger.ok) return ledger;
|
|
2664
|
+
const observation = budgetFromObservation(evidence, ledger.value.budgets.core, now, { resource: "core", source: "core-observer", receivedAt: now, allowEpochChange: true });
|
|
2665
|
+
if (observation.status === "accepted") {
|
|
2666
|
+
ledger.value.budgets.core = observation.budget;
|
|
2667
|
+
ledger.value.epochs.core = observation.budget.epoch;
|
|
2668
|
+
ledger.value.observers.core = { etag: typeof response.etag === "string" ? response.etag : null, outcome: "healthy", at: now, nextAt: now + BUDGET_PROBE_MS };
|
|
2669
|
+
}
|
|
2670
|
+
return writeGovernorState(scope.path, ledger.value);
|
|
2671
|
+
});
|
|
2672
|
+
if (!seeded.ok) return seeded;
|
|
2673
|
+
}
|
|
2674
|
+
if (!isCurrent()) return { ok: false, reason: "stale" };
|
|
2675
|
+
state.identities[credentialKey] = identity;
|
|
2676
|
+
return { ok: true, value: { ...identity, credentialKey } };
|
|
2677
|
+
}, { now });
|
|
2678
|
+
}
|
|
2679
|
+
|
|
2680
|
+
function createSettlementContext(scope, coordinator) {
|
|
2681
|
+
return { scope: { ...scope, identityProvider: null }, isCurrent: () => coordinator.current()?.accessKey === scope.accessKey };
|
|
2682
|
+
}
|
|
2683
|
+
|
|
2684
|
+
function createQuotaScope(identity, { root = identityRegistryRoot(), now = Date.now, identityProvider = null, kill = process.kill.bind(process) } = {}) {
|
|
2685
|
+
return { hash: identity.quotaKey, quotaKey: identity.quotaKey, credentialKey: identity.credentialKey, accessKey: identity.accessKey,
|
|
2686
|
+
path: join(root, `quota-${identity.quotaKey}.json`), coordinationRoot: root, host: identity.host, authIdentity: identity.quotaKey,
|
|
2687
|
+
identityProvider, now, kill };
|
|
2688
|
+
}
|
|
2689
|
+
|
|
2690
|
+
function createIdentityCoordinator({ host, pathOptions = {}, env = process.env, now = Date.now, kill = process.kill.bind(process), runLocalToken, requestIdentity } = {}) {
|
|
2691
|
+
const root = identityRegistryRoot(pathOptions);
|
|
2692
|
+
let current = null;
|
|
2693
|
+
let revision = null;
|
|
2694
|
+
let pending = null;
|
|
2695
|
+
let failure = null;
|
|
2696
|
+
let resolvedCredential = null;
|
|
2697
|
+
let closed = false;
|
|
2698
|
+
const bootstrapAbort = new AbortController();
|
|
2699
|
+
const completions = new Map();
|
|
2700
|
+
const deferCompletion = (key, operation) => completions.set(key, operation);
|
|
2701
|
+
const flushCompletions = async () => {
|
|
2702
|
+
for (const [key, operation] of completions) {
|
|
2703
|
+
const result = await retryIdentityCompletion(operation, { shouldRetry: () => !closed });
|
|
2704
|
+
if (!result.ok && ["busy", "unwritable"].includes(result.reason)) return result;
|
|
2705
|
+
// A malformed/denied proof is a completed durable attempt, not pending
|
|
2706
|
+
// storage work. Drop that callback so ordinary backoff can recover.
|
|
2707
|
+
completions.delete(key);
|
|
2708
|
+
if (!result.ok && result.reason === "corrupt") return result;
|
|
2709
|
+
}
|
|
2710
|
+
return { ok: true };
|
|
2711
|
+
};
|
|
2712
|
+
const resolveHost = () => normalizeHost(typeof host === "function" ? host() : host);
|
|
2713
|
+
const snapshot = () => {
|
|
2714
|
+
if (closed) return null;
|
|
2715
|
+
const selectedHost = resolveHost();
|
|
2716
|
+
if (current && (current.host !== selectedHost || credentialConfigurationRevision(selectedHost, env) !== revision)) current = null;
|
|
2717
|
+
return current;
|
|
2718
|
+
};
|
|
2719
|
+
const refresh = ({ allowBootstrap = true } = {}) => {
|
|
2720
|
+
if (closed) return Promise.resolve({ ok: false, reason: "closed" });
|
|
2721
|
+
if (pending) return pending;
|
|
2722
|
+
pending = (async () => {
|
|
2723
|
+
const completed = await flushCompletions();
|
|
2724
|
+
if (!completed.ok) { failure = completed; return completed; }
|
|
2725
|
+
const selectedHost = resolveHost();
|
|
2726
|
+
const wantedRevision = selectedHost ? credentialConfigurationRevision(selectedHost, env) : null;
|
|
2727
|
+
const resolved = resolvedCredential?.value?.configurationRevision === wantedRevision
|
|
2728
|
+
? resolvedCredential : await resolveEffectiveCredential({ host: selectedHost, env, runLocalToken });
|
|
2729
|
+
if (resolved.ok) resolvedCredential = resolved;
|
|
2730
|
+
if (closed) return { ok: false, reason: "closed" };
|
|
2731
|
+
if (!resolved.ok) { current = null; failure = resolved; return resolved; }
|
|
2732
|
+
revision = resolved.value.configurationRevision;
|
|
2733
|
+
const { credentialKey } = resolved.value;
|
|
2734
|
+
const stored = withIdentityRegistry(root, (state) => {
|
|
2735
|
+
const migration = inspectLegacyMigration(root, state, now());
|
|
2736
|
+
if (!migration.ok) return migration;
|
|
2737
|
+
const identity = state.identities[credentialKey];
|
|
2738
|
+
return { ok: true, value: identity ? { ...identity, credentialKey } : null };
|
|
2739
|
+
}, { now: now(), kill });
|
|
2740
|
+
// A retryable lock failure says nothing about the identity, and nulling
|
|
2741
|
+
// `current` for one makes ensureScope retire the lease and blank every
|
|
2742
|
+
// tab -- discarding the ETags that make the next round cheap.
|
|
2743
|
+
if (!stored.ok) { if (!retryableCoordination(stored.reason)) current = null; failure = stored; return stored; }
|
|
2744
|
+
if (stored.value) { current = stored.value; failure = null; return { ok: true, value: current }; }
|
|
2745
|
+
if (!allowBootstrap) {
|
|
2746
|
+
failure = { ok: false, reason: "identity-unavailable" };
|
|
2747
|
+
return failure;
|
|
2748
|
+
}
|
|
2749
|
+
const claim = claimIdentityBootstrap(root, { credentialKey, host: selectedHost, now: now(), kill });
|
|
2750
|
+
if (!claim.ok) { if (!retryableCoordination(claim.reason)) current = null; failure = claim; return claim; }
|
|
2751
|
+
const proofRevision = resolved.value.configurationRevision;
|
|
2752
|
+
// Check after claim lock waits as well as inside deferred completion:
|
|
2753
|
+
// either wait can outlive the host/configuration that selected this token.
|
|
2754
|
+
const isCurrent = () => !closed && resolveHost() === selectedHost && credentialConfigurationRevision(selectedHost, env) === proofRevision;
|
|
2755
|
+
let response = null;
|
|
2756
|
+
try {
|
|
2757
|
+
if (isCurrent()) response = await (requestIdentity ?? (async (requestHost, { signal }) => {
|
|
2758
|
+
let stdout;
|
|
2759
|
+
try {
|
|
2760
|
+
({ stdout } = await execFileAsync("gh", ["api", "-i", "user", "--hostname", requestHost], {
|
|
2761
|
+
env: { ...env, ...GH_ENV_OVERRIDES }, timeout: GH_TIMEOUT_MS, maxBuffer: GH_MAX_BUFFER, killSignal: "SIGKILL", signal,
|
|
2762
|
+
}));
|
|
2763
|
+
} catch (error) { stdout = error.stdout ?? ""; }
|
|
2764
|
+
const parsed = parseGhApiResponse(stdout);
|
|
2765
|
+
let body = null;
|
|
2766
|
+
try { body = JSON.parse(parsed.body); } catch { /* selected error evidence is still useful */ }
|
|
2767
|
+
return { status: parsed.status, body, rateLimit: pickRateLimit(parsed.headers), etag: parsed.headers.etag ?? null,
|
|
2768
|
+
retryAfter: parsed.headers["retry-after"] ?? null, secondary: SECONDARY_LIMIT_PATTERN.test(String(body?.message ?? "")) };
|
|
2769
|
+
}))(selectedHost, { signal: bootstrapAbort.signal });
|
|
2770
|
+
} catch { /* raw authentication errors never leave the credential seam */ }
|
|
2771
|
+
const finish = () => finishIdentityBootstrap(root, { credentialKey, ...claim.value, response, now: now(), isCurrent });
|
|
2772
|
+
const settled = await retryIdentityCompletion(finish, { shouldRetry: () => !closed });
|
|
2773
|
+
if (!settled.ok && ["busy", "unwritable"].includes(settled.reason)) deferCompletion(`bootstrap:${claim.value.id}`, finish);
|
|
2774
|
+
if (!isCurrent()) { current = null; return { ok: false, reason: "stale" }; }
|
|
2775
|
+
current = settled.ok ? settled.value : null;
|
|
2776
|
+
failure = settled.ok ? null : settled;
|
|
2777
|
+
return settled;
|
|
2778
|
+
})().finally(() => { pending = null; });
|
|
2779
|
+
return pending;
|
|
2780
|
+
};
|
|
2781
|
+
return { root, current: snapshot, refresh, deferCompletion, flushCompletions, inspect: () => failure, isClosed: () => closed, close: () => { closed = true; current = null; bootstrapAbort.abort(); } };
|
|
2782
|
+
}
|
|
2783
|
+
|
|
2784
|
+
async function retryIdentityCompletion(operation, { timeoutMs = GH_TIMEOUT_MS, shouldRetry = () => true } = {}) {
|
|
2785
|
+
const deadline = performance.now() + timeoutMs;
|
|
2786
|
+
for (;;) {
|
|
2787
|
+
const result = operation();
|
|
2788
|
+
if (result.ok || !retryableCoordination(result.reason) || !shouldRetry() || performance.now() >= deadline) return result;
|
|
2789
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
2790
|
+
// Shutdown retains the unresolved receipt; do not keep the CLI alive to
|
|
2791
|
+
// retry a lock owned by another live process. Account switches still retry.
|
|
2792
|
+
if (!shouldRetry()) return result;
|
|
2793
|
+
}
|
|
2794
|
+
}
|
|
2795
|
+
|
|
2796
|
+
function startIdentityControl(coordinator, now = Date.now()) {
|
|
2797
|
+
const identity = coordinator.current();
|
|
2798
|
+
if (!identity) return { ok: false, reason: "identity-unavailable" };
|
|
2799
|
+
return withIdentityRegistry(coordinator.root, (registry) => {
|
|
2800
|
+
const scope = { ...createQuotaScope(identity, { root: coordinator.root, now: () => now }), rootLocked: true };
|
|
2801
|
+
return withGovernorLock(scope, () => {
|
|
2802
|
+
const loaded = readIdentityQuotaState(registry, scope, now);
|
|
2803
|
+
if (!loaded.ok) return loaded;
|
|
2804
|
+
const state = loaded.value;
|
|
2805
|
+
const budget = state.budgets.core;
|
|
2806
|
+
const exceptional = !budget || now >= budget.resetMs + BUDGET_RESET_GRACE_MS;
|
|
2807
|
+
if (budget && now >= budget.resetMs && !exceptional) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
|
|
2808
|
+
if (!exceptional) {
|
|
2809
|
+
const reserved = Object.values(state.reservations).reduce((sum, reservation) => sum + reservationCost(reservation, "core", state.leases, now), 0);
|
|
2810
|
+
if (budget.remaining - resourceReserve(budget.limit) - reserved < 1) return { ok: false, reason: "identity-backoff", retryAt: budget.resetMs + BUDGET_RESET_GRACE_MS };
|
|
2811
|
+
}
|
|
2812
|
+
const recent = Object.values(registry.attempts).filter((attempt) => attempt.credentialKey === identity.credentialKey && attempt.resource === "core" && attempt.exceptional && attempt.startedAt > now - IDENTITY_ATTEMPT_WINDOW_MS);
|
|
2813
|
+
const retryAt = Math.max(0, ...recent.map((attempt) => attempt.retryAt), recent.length >= IDENTITY_ATTEMPT_LIMIT ? Math.min(...recent.map((attempt) => attempt.startedAt)) + IDENTITY_ATTEMPT_WINDOW_MS : 0);
|
|
2814
|
+
if (exceptional && retryAt > now) return { ok: false, reason: "identity-backoff", retryAt };
|
|
2815
|
+
if (Object.keys(registry.attempts).length >= IDENTITY_MAX_ATTEMPTS || Object.keys(state.reservations).length >= GOVERNOR_MAX_RESERVATIONS) return { ok: false, reason: "identity-capacity" };
|
|
2816
|
+
const id = governorId();
|
|
2817
|
+
registry.attempts[id] = { credentialKey: identity.credentialKey, host: identity.host, resource: "core", startedAt: now,
|
|
2818
|
+
retryAt: exceptional ? now + attemptRetryDelayMs(recent.length) : now, ownerPid: process.pid, nonce: governorId(), status: "started",
|
|
2819
|
+
quotaKey: identity.quotaKey, resetMs: budget?.resetMs ?? null, accounted: false, imported: true, exceptional };
|
|
2820
|
+
state.reservations[`reservation:${id}`] = { leaseId: id, intentId: id, costs: { core: 1, graphql: 0 }, actualCosts: null,
|
|
2821
|
+
accountedCosts: { core: 0, graphql: 0 }, notBefore: now, status: "started", epochs: { core: budget?.epoch ?? null, graphql: null },
|
|
2822
|
+
startedAt: now, completedAt: null, outcome: null };
|
|
2823
|
+
const written = writeGovernorState(scope.path, state);
|
|
2824
|
+
return written.ok ? { ok: true, value: { id, identity } } : written;
|
|
2825
|
+
});
|
|
2826
|
+
}, { now });
|
|
2827
|
+
}
|
|
2828
|
+
|
|
2829
|
+
function settleIdentityControl(coordinator, control, stdout, now = Date.now()) {
|
|
2830
|
+
return withIdentityRegistry(coordinator.root, (registry) => {
|
|
2831
|
+
const attempt = registry.attempts[control.id];
|
|
2832
|
+
if (!attempt) return { ok: false, reason: "stale" };
|
|
2833
|
+
const parsed = parseGhApiResponse(stdout ?? "");
|
|
2834
|
+
const evidence = pickRateLimit(parsed.headers);
|
|
2835
|
+
const proven = [200, 304].includes(parsed.status) && evidence?.resource === "core";
|
|
2836
|
+
attempt.status = proven ? "finished" : "uncertain";
|
|
2837
|
+
attempt.accounted = proven;
|
|
2838
|
+
if (evidence?.resource === "core") {
|
|
2839
|
+
attempt.resetMs = evidence.resetMs;
|
|
2840
|
+
if (evidence.remaining === 0) attempt.retryAt = Math.max(attempt.retryAt, evidence.resetMs + BUDGET_RESET_GRACE_MS);
|
|
2841
|
+
}
|
|
2842
|
+
const scope = { ...createQuotaScope(control.identity, { root: coordinator.root, now: () => now }), rootLocked: true };
|
|
2843
|
+
return withGovernorLock(scope, () => {
|
|
2844
|
+
const loaded = readIdentityQuotaState(registry, scope, now);
|
|
2845
|
+
if (!loaded.ok) return loaded;
|
|
2846
|
+
const reservation = loaded.value.reservations[`reservation:${control.id}`];
|
|
2847
|
+
if (!reservation) return { ok: false, reason: "stale" };
|
|
2848
|
+
if (proven) {
|
|
2849
|
+
const cost = parsed.status === 304 ? 0 : 1;
|
|
2850
|
+
reservation.status = "completed";
|
|
2851
|
+
reservation.completedAt = now;
|
|
2852
|
+
reservation.outcome = "measured-success";
|
|
2853
|
+
reservation.actualCosts = { core: cost, graphql: 0 };
|
|
2854
|
+
reservation.accountedCosts = { core: cost, graphql: 0 };
|
|
2855
|
+
}
|
|
2856
|
+
return writeGovernorState(scope.path, loaded.value);
|
|
2857
|
+
});
|
|
2858
|
+
}, { now });
|
|
2859
|
+
}
|
|
2860
|
+
|
|
2861
|
+
async function acquireIdentityHttpPermit(coordinator, { signal, now = Date.now } = {}) {
|
|
2862
|
+
const flushed = await coordinator.flushCompletions?.();
|
|
2863
|
+
if (flushed && !flushed.ok) throw new Error(identityCoordinationMessage(flushed.reason));
|
|
2864
|
+
const identity = coordinator.current();
|
|
2865
|
+
if (!identity) throw new Error("Verified identity unavailable");
|
|
2866
|
+
const queuedAt = now();
|
|
2867
|
+
const deadline = queuedAt + GH_TIMEOUT_MS;
|
|
2868
|
+
const nonce = governorId();
|
|
2869
|
+
let acquired = false;
|
|
2870
|
+
try {
|
|
2871
|
+
for (;;) {
|
|
2872
|
+
if (signal?.aborted) throw new Error("Request cancelled");
|
|
2873
|
+
if (coordinator.current()?.accessKey !== identity.accessKey) throw new Error("Credential changed before request start");
|
|
2874
|
+
if (now() >= deadline) throw new Error(identityCoordinationMessage("transport-busy"));
|
|
2875
|
+
const result = withIdentityRegistry(coordinator.root, (state) => {
|
|
2876
|
+
const migration = inspectLegacyMigration(coordinator.root, state, now());
|
|
2877
|
+
if (!migration.ok) return migration;
|
|
2878
|
+
if (now() >= deadline) return { ok: false, reason: "transport-busy", retryAt: deadline };
|
|
2879
|
+
const transport = state.hosts[identity.host] ??= { lastStartedAt: 0, cooldownUntil: 0, permit: null, waiters: [], throttle: emptyTransportThrottle() };
|
|
2880
|
+
// Five consecutive throttles means the ladder has stopped being useful.
|
|
2881
|
+
// Waiting out the deadline is what produced the last one, so the pause
|
|
2882
|
+
// holds until a person retries or a primary reset opens a recovery.
|
|
2883
|
+
if (transport.throttle?.paused) {
|
|
2884
|
+
return { ok: false, reason: "throttle-paused", retryAt: transport.cooldownUntil };
|
|
2885
|
+
}
|
|
2886
|
+
if (transport.permit && staleHttpPermit(transport.permit, now())) transport.permit = null;
|
|
2887
|
+
// Admission can happen during the mandatory start gap. Persist call
|
|
2888
|
+
// order once so later polling cannot overtake an earlier caller.
|
|
2889
|
+
if (!transport.waiters.some((waiter) => waiter.nonce === nonce)) {
|
|
2890
|
+
if (transport.waiters.length >= HTTP_MAX_WAITERS) return { ok: false, reason: "identity-capacity" };
|
|
2891
|
+
transport.waiters.push({ pid: process.pid, nonce, queuedAt, deadline });
|
|
2892
|
+
}
|
|
2893
|
+
const retryAt = Math.max(transport.cooldownUntil, transport.lastStartedAt + HTTP_START_GAP_MS);
|
|
2894
|
+
if (transport.permit || transport.waiters[0].nonce !== nonce || retryAt > now()) {
|
|
2895
|
+
return { ok: false, reason: "transport-busy", retryAt: Math.max(retryAt, now() + 50) };
|
|
2896
|
+
}
|
|
2897
|
+
transport.waiters.shift();
|
|
2898
|
+
transport.permit = { pid: process.pid, nonce, startedAt: now() };
|
|
2899
|
+
transport.lastStartedAt = now();
|
|
2900
|
+
return { ok: true, value: { nonce, host: identity.host } };
|
|
2901
|
+
}, { now: now() });
|
|
2902
|
+
if (result.ok) { acquired = true; return result.value; }
|
|
2903
|
+
// A contended registry lock is the same class of wait as a held permit:
|
|
2904
|
+
// both mean "not yet", not "no identity". Only transport-busy carries a
|
|
2905
|
+
// retryAt, so give the rest the poll interval the loop already uses.
|
|
2906
|
+
const retryAt = result.retryAt ?? now() + 50;
|
|
2907
|
+
if ((result.reason !== "transport-busy" && !retryableCoordination(result.reason)) ||
|
|
2908
|
+
now() >= deadline || retryAt >= deadline) throw new Error(identityCoordinationMessage(result.reason));
|
|
2909
|
+
await new Promise((resolve) => setTimeout(resolve, Math.max(1, Math.min(50, retryAt - now()))));
|
|
2910
|
+
}
|
|
2911
|
+
} finally {
|
|
2912
|
+
if (!acquired) {
|
|
2913
|
+
const cleanup = () => withIdentityRegistry(coordinator.root, (state) => {
|
|
2914
|
+
const transport = state.hosts[identity.host];
|
|
2915
|
+
if (transport) transport.waiters = transport.waiters.filter((waiter) => waiter.nonce !== nonce);
|
|
2916
|
+
return { ok: true };
|
|
2917
|
+
}, { now: now() });
|
|
2918
|
+
const cleaned = cleanup();
|
|
2919
|
+
if (!cleaned.ok) coordinator.deferCompletion?.(`waiter:${nonce}`, cleanup);
|
|
2920
|
+
}
|
|
2921
|
+
}
|
|
2922
|
+
}
|
|
2923
|
+
|
|
2924
|
+
// One cooldown law, two call sites: the bootstrap observer reads its evidence
|
|
2925
|
+
// from a parsed response, the permit release from a failed subprocess's captured
|
|
2926
|
+
// output. They had already drifted into two implementations with two different
|
|
2927
|
+
// secondary-limit detectors -- which is how a tuning applied to one quietly
|
|
2928
|
+
// stops applying to the other.
|
|
2929
|
+
const SECONDARY_LIMIT_PATTERN = /secondary rate|abuse/i;
|
|
2930
|
+
|
|
2931
|
+
// The locally chosen delay climbs by consecutive throttle and stops climbing at
|
|
2932
|
+
// fifteen minutes. The cap bounds only what this client invents for itself: a
|
|
2933
|
+
// server that asks for two hours is given two hours, because it knows something
|
|
2934
|
+
// we do not.
|
|
2935
|
+
const THROTTLE_LADDER_MS = [60_000, 120_000, 240_000, 480_000, 900_000];
|
|
2936
|
+
// After this many consecutive throttles the transport stops choosing new delays
|
|
2937
|
+
// and waits for a person or a primary reset. Climbing forever keeps a wedged
|
|
2938
|
+
// credential politely hammering a limit it has no way to satisfy.
|
|
2939
|
+
const THROTTLE_PAUSE_AFTER = 5;
|
|
2940
|
+
|
|
2941
|
+
function throttleLadderMs(priorThrottles) {
|
|
2942
|
+
const step = Number.isSafeInteger(priorThrottles) && priorThrottles > 0 ? priorThrottles : 0;
|
|
2943
|
+
return THROTTLE_LADDER_MS[Math.min(step, THROTTLE_LADDER_MS.length - 1)];
|
|
2944
|
+
}
|
|
2945
|
+
|
|
2946
|
+
// Reads throttle evidence without inventing any. The case this exists to get
|
|
2947
|
+
// right is the permission-only 403: it carries the same status as an exhausted
|
|
2948
|
+
// primary limit and is not a throttle at all, so treating it as one pauses a
|
|
2949
|
+
// pane that was never limited -- it was just not allowed.
|
|
2950
|
+
function classifyThrottle({ status = null, headers = null, graphqlErrors = null, stderr = "", body = null } = {}) {
|
|
2951
|
+
const header = (name) => {
|
|
2952
|
+
const value = headers?.[name];
|
|
2953
|
+
return typeof value === "string" ? value.trim() : null;
|
|
2954
|
+
};
|
|
2955
|
+
const integer = (name) => {
|
|
2956
|
+
const value = Number(header(name));
|
|
2957
|
+
return Number.isSafeInteger(value) ? value : null;
|
|
2958
|
+
};
|
|
2959
|
+
const secondary = SECONDARY_LIMIT_PATTERN.test([
|
|
2960
|
+
String(body?.message ?? ""),
|
|
2961
|
+
String(stderr ?? ""),
|
|
2962
|
+
...(Array.isArray(graphqlErrors)
|
|
2963
|
+
? graphqlErrors.map((error) => `${error?.type ?? ""} ${error?.message ?? ""}`)
|
|
2964
|
+
: []),
|
|
2965
|
+
].join("\n"));
|
|
2966
|
+
const retryAfter = header("retry-after");
|
|
2967
|
+
const remaining = integer("x-ratelimit-remaining");
|
|
2968
|
+
const reset = integer("x-ratelimit-reset");
|
|
2969
|
+
const resource = header("x-ratelimit-resource")?.toLowerCase() ?? null;
|
|
2970
|
+
// The counter itself says there is nothing left. That holds its own resource
|
|
2971
|
+
// until its own reset, and says nothing about the other one.
|
|
2972
|
+
if (!secondary && remaining === 0 && reset !== null && RATE_RESOURCES.includes(resource) &&
|
|
2973
|
+
(status === 403 || status === 429)) {
|
|
2974
|
+
return { kind: "primary", resource, resetMs: reset * 1000 };
|
|
2975
|
+
}
|
|
2976
|
+
if (retryAfter !== null || secondary || status === 429) {
|
|
2977
|
+
return { kind: "secondary", retryAfter, secondary, status };
|
|
2978
|
+
}
|
|
2979
|
+
return { kind: "none" };
|
|
2980
|
+
}
|
|
2981
|
+
|
|
2982
|
+
function transportCooldownDeadline({ retryAfter, status, secondary, at, attempts = 0 }) {
|
|
2983
|
+
const supplied = typeof retryAfter === "string" && /^\d+(?:\.\d+)?$/.test(retryAfter)
|
|
2984
|
+
? at + Number(retryAfter) * 1000
|
|
2985
|
+
: Date.parse(retryAfter ?? "");
|
|
2986
|
+
// A deadline the server supplied is honoured exactly, however long. Only the
|
|
2987
|
+
// delay chosen here is laddered, and only it is capped.
|
|
2988
|
+
if (Number.isFinite(supplied)) return supplied;
|
|
2989
|
+
return status === 429 || secondary === true ? at + throttleLadderMs(attempts) : null;
|
|
2990
|
+
}
|
|
2991
|
+
|
|
2992
|
+
function emptyTransportThrottle() {
|
|
2993
|
+
return { attempts: 0, lastAt: 0, paused: false };
|
|
2994
|
+
}
|
|
2995
|
+
|
|
2996
|
+
function applyTransportCooldown(transport, evidence) {
|
|
2997
|
+
const throttle = transport.throttle ?? emptyTransportThrottle();
|
|
2998
|
+
const deadline = transportCooldownDeadline({ ...evidence, attempts: throttle.attempts });
|
|
2999
|
+
if (deadline === null) return null;
|
|
3000
|
+
// Merged by maximum, so a shorter concurrent error never shortens a hold
|
|
3001
|
+
// another response already established, and the hold survives a primary epoch
|
|
3002
|
+
// change: secondary limits are not accounted in the primary counter.
|
|
3003
|
+
transport.cooldownUntil = Math.max(transport.cooldownUntil, deadline);
|
|
3004
|
+
const attempts = Math.min(throttle.attempts + 1, THROTTLE_PAUSE_AFTER);
|
|
3005
|
+
transport.throttle = { attempts, lastAt: evidence.at, paused: attempts >= THROTTLE_PAUSE_AFTER };
|
|
3006
|
+
return transport.cooldownUntil;
|
|
3007
|
+
}
|
|
3008
|
+
|
|
3009
|
+
// A request that came back carrying no throttle evidence is the only thing that
|
|
3010
|
+
// proves the hold is over, so it is the only thing that resets the ladder.
|
|
3011
|
+
// Merely waiting out a deadline does not: that is what produced the next
|
|
3012
|
+
// throttle last time.
|
|
3013
|
+
function clearTransportThrottle(transport) {
|
|
3014
|
+
if (transport && transport.throttle && transport.throttle.attempts > 0) {
|
|
3015
|
+
transport.throttle = emptyTransportThrottle();
|
|
3016
|
+
}
|
|
3017
|
+
}
|
|
3018
|
+
|
|
3019
|
+
function releaseIdentityHttpPermit(coordinator, permit, error = null) {
|
|
3020
|
+
return withIdentityRegistry(coordinator.root, (state) => {
|
|
3021
|
+
const transport = state.hosts[permit.host];
|
|
3022
|
+
if (transport?.permit?.nonce === permit.nonce) transport.permit = null;
|
|
3023
|
+
if (transport && error) {
|
|
3024
|
+
const response = parseGhApiResponse(error.stdout ?? "");
|
|
3025
|
+
const verdict = classifyThrottle({
|
|
3026
|
+
status: response.status,
|
|
3027
|
+
headers: response.headers,
|
|
3028
|
+
stderr: error.stderr,
|
|
3029
|
+
});
|
|
3030
|
+
// A permission-only 403 reaches here too. It is a failure, but it is not a
|
|
3031
|
+
// throttle, and holding the shared transport for it would pause every
|
|
3032
|
+
// pane over one repository the user cannot read.
|
|
3033
|
+
if (verdict.kind === "secondary") {
|
|
3034
|
+
applyTransportCooldown(transport, {
|
|
3035
|
+
retryAfter: verdict.retryAfter,
|
|
3036
|
+
status: verdict.status,
|
|
3037
|
+
secondary: verdict.secondary,
|
|
3038
|
+
at: Date.now(),
|
|
3039
|
+
});
|
|
3040
|
+
}
|
|
3041
|
+
} else if (transport && !error) {
|
|
3042
|
+
clearTransportThrottle(transport);
|
|
3043
|
+
}
|
|
3044
|
+
return { ok: true };
|
|
3045
|
+
});
|
|
3046
|
+
}
|
|
3047
|
+
|
|
3048
|
+
// The explicit retry a paused transport waits for. Clears the ladder and the
|
|
3049
|
+
// deadline together: a person choosing to retry is choosing to spend the next
|
|
3050
|
+
// request finding out, which is exactly the recovery election the pause defers.
|
|
3051
|
+
function retryThrottledTransport(root, host, { now = Date.now } = {}) {
|
|
3052
|
+
return withIdentityRegistry(root, (state) => {
|
|
3053
|
+
const transport = state.hosts[host];
|
|
3054
|
+
if (!transport?.throttle?.paused) return { ok: true, value: { resumed: false } };
|
|
3055
|
+
transport.throttle = emptyTransportThrottle();
|
|
3056
|
+
transport.cooldownUntil = Math.min(transport.cooldownUntil, now());
|
|
3057
|
+
return { ok: true, value: { resumed: true } };
|
|
3058
|
+
}, { now: now() });
|
|
3059
|
+
}
|
|
3060
|
+
|
|
3061
|
+
function identityCoordinationMessage(reason) {
|
|
3062
|
+
if (reason === "restart-required") return "Restart required: close older gh-glance panes";
|
|
3063
|
+
if (["migration-hold", "legacy-unresolved"].includes(reason)) return "Upgrade waiting for legacy quota reset";
|
|
3064
|
+
if (["legacy-corrupt", "corrupt"].includes(reason)) return "Coordination state unavailable; evidence preserved";
|
|
3065
|
+
if (reason === "transport-busy") return "Shared HTTP request or cooldown in progress";
|
|
3066
|
+
if (reason === "throttle-paused") return "Paused after repeated rate limits; refresh to retry";
|
|
3067
|
+
if (retryableCoordination(reason)) return "Coordinating with your other panes";
|
|
3068
|
+
if (reason === "identity-capacity") return "Coordination state full; retrying after the current window";
|
|
3069
|
+
return "Verified identity unavailable; waiting to retry";
|
|
3070
|
+
}
|
|
3071
|
+
|
|
1859
3072
|
// ---------- Shared account governor ----------
|
|
1860
3073
|
|
|
1861
3074
|
// The scope version is deliberately stable. Stored protocol revisions must use
|
|
1862
3075
|
// the same file so an older process sees the new exact shape and fails closed
|
|
1863
3076
|
// instead of coordinating through a second file.
|
|
1864
3077
|
const GOVERNOR_SCOPE_VERSION = 1;
|
|
1865
|
-
|
|
3078
|
+
// 3: GraphQL budgets carry `graphql-observer` provenance. An older build does
|
|
3079
|
+
// not know that source and would reject the budget -- and normalizeGovernorState
|
|
3080
|
+
// turns one rejected budget into total loss, discarding every live pane's
|
|
3081
|
+
// leases, intents and reservations. The version gate makes it fail closed on the
|
|
3082
|
+
// file instead, which is the whole point of having one.
|
|
3083
|
+
// 4: observer claims, readiness and outcomes are per resource, so the schema
|
|
3084
|
+
// shape changed rather than a field's contents. Replaced atomically -- an older
|
|
3085
|
+
// build must fail closed on the version gate rather than half-read it.
|
|
3086
|
+
// 5: scheduling fairness is persisted, because the starvation it prevents
|
|
3087
|
+
// happens *between* planning passes -- each manual refresh is its own pass, so
|
|
3088
|
+
// a counter that lives only inside one cannot see a run of them.
|
|
3089
|
+
const GOVERNOR_STATE_VERSION = 5;
|
|
3090
|
+
// Readable-as-evidence, never written: the shapes a still-running older pane
|
|
3091
|
+
// holds. Recognising that such a pane owns a live lease is what the restart
|
|
3092
|
+
// boundary depends on.
|
|
3093
|
+
const LEGACY_GOVERNOR_VERSIONS = [GOVERNOR_STATE_VERSION, 4, 3, 2];
|
|
3094
|
+
// The first version whose file is already in the per-resource observer shape.
|
|
3095
|
+
// Anything below it is adapted before it can be read; see readLegacyGovernorState.
|
|
3096
|
+
const GOVERNOR_OBSERVER_SPLIT_VERSION = 4;
|
|
3097
|
+
// The first version carrying persisted scheduling fairness. A version 4 file is
|
|
3098
|
+
// otherwise in the current shape and only wants the field defaulted.
|
|
3099
|
+
const GOVERNOR_FAIRNESS_VERSION = 5;
|
|
3100
|
+
// Manual work outranks active work, which is right until it means "always". A
|
|
3101
|
+
// user holding the refresh key issues one manual intent per planning pass, and
|
|
3102
|
+
// without a bound the active owner behind them never gets a turn at all.
|
|
3103
|
+
const MANUAL_GRANT_STREAK_LIMIT = 3;
|
|
1866
3104
|
const GOVERNOR_MAX_LEASES = 128;
|
|
1867
3105
|
const GOVERNOR_MAX_INTENTS = 512;
|
|
1868
3106
|
const GOVERNOR_MAX_RESERVATIONS = 512;
|
|
@@ -1965,7 +3203,8 @@ function currentGovernorScope(scope) {
|
|
|
1965
3203
|
} catch {
|
|
1966
3204
|
return { ok: false, reason: "stale" };
|
|
1967
3205
|
}
|
|
1968
|
-
const hash = governorScopeHash(current?.effectiveHost, current?.authIdentity);
|
|
3206
|
+
const hash = scope.quotaKey ? current?.quotaKey : governorScopeHash(current?.effectiveHost, current?.authIdentity);
|
|
3207
|
+
if (scope.quotaKey && current?.accessKey !== scope.accessKey) return { ok: false, reason: "stale" };
|
|
1969
3208
|
return hash === scope.hash ? { ok: true, value: scope } : { ok: false, reason: "stale" };
|
|
1970
3209
|
}
|
|
1971
3210
|
|
|
@@ -1974,11 +3213,16 @@ function emptyGovernorState() {
|
|
|
1974
3213
|
version: GOVERNOR_STATE_VERSION,
|
|
1975
3214
|
epochs: { core: null, graphql: null },
|
|
1976
3215
|
budgets: {},
|
|
1977
|
-
|
|
1978
|
-
|
|
1979
|
-
|
|
1980
|
-
probeClaim
|
|
1981
|
-
|
|
3216
|
+
fairness: { manualStreak: 0 },
|
|
3217
|
+
// Symmetric on purpose. Core observer state lived in `observers.core` while
|
|
3218
|
+
// the GraphQL one lived in a differently-shaped `probeOutcome`, and a single
|
|
3219
|
+
// `probeClaim` serialized both -- so a slow or failed observer for one
|
|
3220
|
+
// resource held up the other's readiness for no reason the resources
|
|
3221
|
+
// themselves impose.
|
|
3222
|
+
observers: Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
3223
|
+
resource, { etag: null, outcome: "idle", at: 0, nextAt: 0 },
|
|
3224
|
+
])),
|
|
3225
|
+
probeClaims: Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, null])),
|
|
1982
3226
|
leases: {},
|
|
1983
3227
|
intents: {},
|
|
1984
3228
|
reservations: {},
|
|
@@ -2013,8 +3257,12 @@ function normalizeGovernorBudget(raw, nowMs, resource = null) {
|
|
|
2013
3257
|
(raw.blockReason !== null && !GOVERNOR_BLOCK_REASONS.has(raw.blockReason)) ||
|
|
2014
3258
|
(raw.roundRobinCursor !== null && !validGovernorId(raw.roundRobinCursor)) ||
|
|
2015
3259
|
!Number.isFinite(raw.lastExternalFactor) || raw.lastExternalFactor < 1 ||
|
|
2016
|
-
!["response-header", "core-observer", "rate-limit-probe"].includes(raw.source) ||
|
|
2017
|
-
|
|
3260
|
+
!["response-header", "core-observer", "graphql-observer", "rate-limit-probe"].includes(raw.source) ||
|
|
3261
|
+
// "rate-limit-probe" is retained as readable legacy provenance only: ledgers
|
|
3262
|
+
// written before the claimed GraphQL observer existed still carry it. Nothing
|
|
3263
|
+
// writes it any more, and neither it nor the GraphQL observer may ever stand
|
|
3264
|
+
// as core authority.
|
|
3265
|
+
["rate-limit-probe", "graphql-observer"].includes(raw.source) && resource === "core" ||
|
|
2018
3266
|
!exactKeys(raw.factorBaseline, ["epoch", "used", "observedAt"]) ||
|
|
2019
3267
|
!validGovernorEpoch(raw.factorBaseline.epoch) ||
|
|
2020
3268
|
!Number.isSafeInteger(raw.factorBaseline.used) || raw.factorBaseline.used < 0 ||
|
|
@@ -2098,7 +3346,7 @@ function normalizeGovernorReservation(raw, nowMs) {
|
|
|
2098
3346
|
return { ...raw, costs, actualCosts, accountedCosts, epochs };
|
|
2099
3347
|
}
|
|
2100
3348
|
|
|
2101
|
-
function
|
|
3349
|
+
function normalizeResourceObserver(raw, nowMs) {
|
|
2102
3350
|
if (!exactKeys(raw, ["etag", "outcome", "at", "nextAt"])) return null;
|
|
2103
3351
|
if (
|
|
2104
3352
|
raw.etag !== null && (typeof raw.etag !== "string" || raw.etag.length === 0) ||
|
|
@@ -2112,24 +3360,19 @@ function normalizeCoreObserver(raw, nowMs) {
|
|
|
2112
3360
|
function normalizeProbeClaim(raw, nowMs) {
|
|
2113
3361
|
if (raw === null) return null;
|
|
2114
3362
|
if (!exactKeys(raw, [
|
|
2115
|
-
"ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds",
|
|
3363
|
+
"ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds",
|
|
2116
3364
|
])) {
|
|
2117
3365
|
return undefined;
|
|
2118
3366
|
}
|
|
2119
|
-
const resources = Array.isArray(raw.resources) && raw.resources.length > 0 &&
|
|
2120
|
-
raw.resources.every((resource) => RATE_RESOURCES.includes(resource))
|
|
2121
|
-
? [...new Set(raw.resources)]
|
|
2122
|
-
: null;
|
|
2123
3367
|
if (
|
|
2124
3368
|
!validGovernorId(raw.ownerLeaseId) || !validGovernorId(raw.nonce) ||
|
|
2125
3369
|
finiteTimestamp(raw.leaseUntil, nowMs) === undefined ||
|
|
2126
3370
|
finiteTimestamp(raw.nextAt, nowMs) === undefined ||
|
|
2127
3371
|
finiteTimestamp(raw.claimAt, nowMs) === undefined || raw.claimAt > nowMs ||
|
|
2128
3372
|
!Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
|
|
2129
|
-
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
|
|
2130
|
-
!resources
|
|
3373
|
+
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
|
|
2131
3374
|
) return undefined;
|
|
2132
|
-
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)]
|
|
3375
|
+
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
|
|
2133
3376
|
}
|
|
2134
3377
|
|
|
2135
3378
|
function normalizeManualProbe(raw, nowMs) {
|
|
@@ -2144,29 +3387,34 @@ function normalizeManualProbe(raw, nowMs) {
|
|
|
2144
3387
|
return { ...raw };
|
|
2145
3388
|
}
|
|
2146
3389
|
|
|
2147
|
-
|
|
3390
|
+
// `acceptVersion` exists for one caller: the legacy inspector, which must be
|
|
3391
|
+
// able to *read* a still-running older pane's file to see its live leases. It
|
|
3392
|
+
// never publishes what it reads. Every other caller takes the default, so a
|
|
3393
|
+
// version this build does not write is unreadable and fails closed.
|
|
3394
|
+
function normalizeGovernorState(raw, nowMs, { prune = true, acceptVersion = GOVERNOR_STATE_VERSION } = {}) {
|
|
2148
3395
|
if (!exactKeys(raw, [
|
|
2149
|
-
"version", "epochs", "budgets", "
|
|
3396
|
+
"version", "epochs", "budgets", "fairness", "observers", "probeClaims", "leases",
|
|
2150
3397
|
"intents", "reservations", "manualProbe",
|
|
2151
|
-
]) || raw.version !==
|
|
3398
|
+
]) || raw.version !== acceptVersion) return null;
|
|
3399
|
+
if (!exactKeys(raw.fairness, ["manualStreak"]) ||
|
|
3400
|
+
!Number.isSafeInteger(raw.fairness.manualStreak) || raw.fairness.manualStreak < 0) return null;
|
|
2152
3401
|
if (
|
|
2153
3402
|
!exactKeys(raw.epochs, RATE_RESOURCES) ||
|
|
2154
3403
|
RATE_RESOURCES.some((resource) => raw.epochs[resource] !== null && !validGovernorEpoch(raw.epochs[resource])) ||
|
|
2155
3404
|
!isRecord(raw.budgets) || Object.keys(raw.budgets).some((resource) => !RATE_RESOURCES.includes(resource)) ||
|
|
2156
|
-
!exactKeys(raw.observers,
|
|
2157
|
-
!
|
|
2158
|
-
!
|
|
2159
|
-
!GOVERNOR_PROBE_STATUSES.has(raw.probeOutcome.status) ||
|
|
2160
|
-
finiteTimestamp(raw.probeOutcome.at, nowMs) === undefined ||
|
|
2161
|
-
raw.probeOutcome.at > nowMs ||
|
|
2162
|
-
finiteTimestamp(raw.probeOutcome.nextAt, nowMs) === undefined
|
|
3405
|
+
!exactKeys(raw.observers, RATE_RESOURCES) ||
|
|
3406
|
+
!exactKeys(raw.probeClaims, RATE_RESOURCES) ||
|
|
3407
|
+
!isRecord(raw.leases) || !isRecord(raw.intents) || !isRecord(raw.reservations)
|
|
2163
3408
|
) return null;
|
|
2164
3409
|
|
|
2165
3410
|
const state = emptyGovernorState();
|
|
2166
3411
|
state.epochs = { ...raw.epochs };
|
|
2167
|
-
state.
|
|
2168
|
-
|
|
2169
|
-
|
|
3412
|
+
state.fairness = { ...raw.fairness };
|
|
3413
|
+
for (const resource of RATE_RESOURCES) {
|
|
3414
|
+
const observer = normalizeResourceObserver(raw.observers[resource], nowMs);
|
|
3415
|
+
if (!observer) return null;
|
|
3416
|
+
state.observers[resource] = observer;
|
|
3417
|
+
}
|
|
2170
3418
|
for (const resource of Object.keys(raw.budgets)) {
|
|
2171
3419
|
const budget = normalizeGovernorBudget(raw.budgets[resource], nowMs, resource);
|
|
2172
3420
|
if (!budget) return null;
|
|
@@ -2196,21 +3444,75 @@ function normalizeGovernorState(raw, nowMs, { prune = true } = {}) {
|
|
|
2196
3444
|
) state.reservations[id] = reservation;
|
|
2197
3445
|
}
|
|
2198
3446
|
if (Object.keys(state.reservations).length > GOVERNOR_MAX_RESERVATIONS) return null;
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
3447
|
+
for (const resource of RATE_RESOURCES) {
|
|
3448
|
+
const claim = normalizeProbeClaim(raw.probeClaims[resource], nowMs);
|
|
3449
|
+
if (claim === undefined) return null;
|
|
3450
|
+
state.probeClaims[resource] = claim?.leaseUntil > nowMs ? claim : null;
|
|
3451
|
+
}
|
|
2202
3452
|
state.manualProbe = normalizeManualProbe(raw.manualProbe, nowMs);
|
|
2203
3453
|
if (state.manualProbe === undefined) return null;
|
|
2204
3454
|
return state;
|
|
2205
3455
|
}
|
|
2206
3456
|
|
|
3457
|
+
// Reads a still-running older pane's file as evidence. A version number alone
|
|
3458
|
+
// is not enough to identify a document: versions 2 and 3 share a *shape* that
|
|
3459
|
+
// version 4 replaced, so accepting their version against the current key set
|
|
3460
|
+
// would reject every real file they wrote. Read-only -- the result is never
|
|
3461
|
+
// published and the file is never rewritten.
|
|
3462
|
+
function readLegacyGovernorState(raw, nowMs, version) {
|
|
3463
|
+
if (!isRecord(raw) || raw.version !== version) return null;
|
|
3464
|
+
const options = { prune: false, acceptVersion: version };
|
|
3465
|
+
if (version >= GOVERNOR_FAIRNESS_VERSION) return normalizeGovernorState(raw, nowMs, options);
|
|
3466
|
+
// A version 4 file is already in the per-resource observer shape and differs
|
|
3467
|
+
// only by the field this version added, so defaulting it is the whole
|
|
3468
|
+
// adaptation. Older files need their observer shape rewritten first.
|
|
3469
|
+
const shaped = version >= GOVERNOR_OBSERVER_SPLIT_VERSION ? { ...raw } : adaptPreSplitGovernorShape(raw);
|
|
3470
|
+
if (!shaped) return null;
|
|
3471
|
+
shaped.fairness = { manualStreak: 0 };
|
|
3472
|
+
return normalizeGovernorState(shaped, nowMs, options);
|
|
3473
|
+
}
|
|
3474
|
+
|
|
3475
|
+
// Versions 2 and 3 held one observer for core, a differently shaped
|
|
3476
|
+
// `probeOutcome` standing in for GraphQL, and a single claim naming the
|
|
3477
|
+
// resources it covered. Rewrite that into the current shape so the inspector
|
|
3478
|
+
// above can see the leases, deadlines and uncertain work such a pane holds.
|
|
3479
|
+
// The claim's own `resources` list says which resources it covered, so
|
|
3480
|
+
// splitting it in two reads the file rather than guessing at it.
|
|
3481
|
+
function adaptPreSplitGovernorShape(raw) {
|
|
3482
|
+
if (!exactKeys(raw, [
|
|
3483
|
+
"version", "epochs", "budgets", "observers", "probeClaim", "probeOutcome",
|
|
3484
|
+
"leases", "intents", "reservations", "manualProbe",
|
|
3485
|
+
])) return null;
|
|
3486
|
+
if (!exactKeys(raw.observers, ["core"])) return null;
|
|
3487
|
+
if (!exactKeys(raw.probeOutcome, ["status", "at", "nextAt"])) return null;
|
|
3488
|
+
if (raw.probeClaim !== null && !isRecord(raw.probeClaim)) return null;
|
|
3489
|
+
const { resources, ...claim } = raw.probeClaim ?? {};
|
|
3490
|
+
const covered = Array.isArray(resources) ? resources : [];
|
|
3491
|
+
const adapted = structuredClone(raw);
|
|
3492
|
+
delete adapted.probeClaim;
|
|
3493
|
+
delete adapted.probeOutcome;
|
|
3494
|
+
adapted.observers = {
|
|
3495
|
+
core: raw.observers.core,
|
|
3496
|
+
graphql: {
|
|
3497
|
+
etag: null,
|
|
3498
|
+
outcome: raw.probeOutcome.status,
|
|
3499
|
+
at: raw.probeOutcome.at,
|
|
3500
|
+
nextAt: raw.probeOutcome.nextAt,
|
|
3501
|
+
},
|
|
3502
|
+
};
|
|
3503
|
+
adapted.probeClaims = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
3504
|
+
resource, raw.probeClaim && covered.includes(resource) ? structuredClone(claim) : null,
|
|
3505
|
+
]));
|
|
3506
|
+
return adapted;
|
|
3507
|
+
}
|
|
3508
|
+
|
|
2207
3509
|
function serializeGovernorState(state) {
|
|
2208
3510
|
return `${JSON.stringify(state)}\n`;
|
|
2209
3511
|
}
|
|
2210
3512
|
|
|
2211
3513
|
function migrateGovernorState(raw, nowMs) {
|
|
2212
3514
|
if (!exactKeys(raw, [
|
|
2213
|
-
"version", "epochs", "budgets", "probeClaim", "probeOutcome", "leases",
|
|
3515
|
+
"version", "epochs", "budgets", "probeClaim", "probeOutcome", "leases", // v1 shape
|
|
2214
3516
|
"intents", "reservations", "manualProbe",
|
|
2215
3517
|
]) || raw.version !== 1 || !isRecord(raw.budgets) || !isRecord(raw.reservations)) return null;
|
|
2216
3518
|
const migrated = structuredClone(raw);
|
|
@@ -2220,20 +3522,28 @@ function migrateGovernorState(raw, nowMs) {
|
|
|
2220
3522
|
// preserve it as bootstrap evidence. GraphQL remains valid.
|
|
2221
3523
|
delete migrated.budgets.core;
|
|
2222
3524
|
migrated.epochs.core = null;
|
|
2223
|
-
|
|
2224
|
-
|
|
2225
|
-
|
|
2226
|
-
|
|
2227
|
-
|
|
2228
|
-
|
|
2229
|
-
|
|
2230
|
-
|
|
2231
|
-
|
|
2232
|
-
}
|
|
3525
|
+
// The v1 GraphQL value came from /rate_limit, for the same reason the core
|
|
3526
|
+
// value did, and is dropped for the same reason: this phase makes that
|
|
3527
|
+
// endpoint explicitly non-authoritative, and "never a source of spendable
|
|
3528
|
+
// capacity" cannot have an exception for numbers that arrived before the rule.
|
|
3529
|
+
// Keeping it would let a migrated pane admit GraphQL work against it and let
|
|
3530
|
+
// it establish the epoch later headers refine. The claimed observer
|
|
3531
|
+
// re-establishes capacity on the next probe.
|
|
3532
|
+
delete migrated.budgets.graphql;
|
|
3533
|
+
migrated.epochs.graphql = null;
|
|
2233
3534
|
for (const reservation of Object.values(migrated.reservations)) {
|
|
2234
3535
|
reservation.accountedCosts = { core: 0, graphql: 0 };
|
|
2235
3536
|
}
|
|
2236
|
-
|
|
3537
|
+
// v1 carried one claim for both resources; the new shape has one per resource
|
|
3538
|
+
// and a migrated claim belongs to neither, so it is dropped rather than
|
|
3539
|
+
// guessed at. The next probe re-claims what is actually due.
|
|
3540
|
+
delete migrated.probeClaim;
|
|
3541
|
+
delete migrated.probeOutcome;
|
|
3542
|
+
migrated.observers = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
3543
|
+
resource, { etag: null, outcome: "idle", at: 0, nextAt: 0 },
|
|
3544
|
+
]));
|
|
3545
|
+
migrated.probeClaims = Object.fromEntries(RATE_RESOURCES.map((resource) => [resource, null]));
|
|
3546
|
+
migrated.fairness = { manualStreak: 0 };
|
|
2237
3547
|
return normalizeGovernorState(migrated, nowMs);
|
|
2238
3548
|
}
|
|
2239
3549
|
|
|
@@ -2410,6 +3720,16 @@ function withGovernorLock(scope, operation, {
|
|
|
2410
3720
|
kill = scope?.kill ?? process.kill.bind(process),
|
|
2411
3721
|
observeArtifact = null,
|
|
2412
3722
|
} = {}) {
|
|
3723
|
+
if (scope?.coordinationRoot && !scope.rootLocked) {
|
|
3724
|
+
return withIdentityRegistry(scope.coordinationRoot, (registry, at) => {
|
|
3725
|
+
const migration = inspectLegacyMigration(scope.coordinationRoot, registry, at);
|
|
3726
|
+
if (!migration.ok) return migration;
|
|
3727
|
+
if (registry.identities[scope.credentialKey]) {
|
|
3728
|
+
try { statSync(scope.path); } catch { return { ok: false, reason: "corrupt" }; }
|
|
3729
|
+
}
|
|
3730
|
+
return withGovernorLock({ ...scope, rootLocked: true }, operation, { pid, nonce, waitMs, kill, observeArtifact });
|
|
3731
|
+
}, { now: scopeNow(scope), kill });
|
|
3732
|
+
}
|
|
2413
3733
|
const current = currentGovernorScope(scope);
|
|
2414
3734
|
if (!current.ok) return current;
|
|
2415
3735
|
const lockPath = `${scope.path}.lock`;
|
|
@@ -2506,6 +3826,33 @@ function mutateGovernor(scope, nowMs, mutate) {
|
|
|
2506
3826
|
});
|
|
2507
3827
|
}
|
|
2508
3828
|
|
|
3829
|
+
// The largest single operation the governor will ever admit. Returned pacing is
|
|
3830
|
+
// capped at one of these so an idle stretch cannot accumulate into a burst: at
|
|
3831
|
+
// most one operation's worth of the lane is ever given back at a time.
|
|
3832
|
+
const GOVERNOR_MAX_ATOMIC_COST = Math.max(...Object.values(OPERATION_COSTS)
|
|
3833
|
+
.flatMap((costs) => RATE_RESOURCES.map((resource) => costs?.[resource] ?? 0)));
|
|
3834
|
+
|
|
3835
|
+
// A grant paces the lane by what it *reserved*, which is a worst case. When the
|
|
3836
|
+
// request costs less than that -- a 304 costs nothing -- or never happens at
|
|
3837
|
+
// all, the difference is capacity paced away for nobody, and the next request
|
|
3838
|
+
// waits out a slot no one used.
|
|
3839
|
+
//
|
|
3840
|
+
// The lane is never pulled earlier than the transport gap, which is what keeps
|
|
3841
|
+
// this from becoming a burst. Recomputing the rate here can differ from the
|
|
3842
|
+
// rate at grant time if capacity has since changed, so the returned credit is
|
|
3843
|
+
// an estimate rather than an exact reversal; that is safe because admission
|
|
3844
|
+
// still re-checks affordability against the reserve before anything starts.
|
|
3845
|
+
// Pacing decides when work may go, never whether it may.
|
|
3846
|
+
function returnPacingCredit(state, resource, unusedCost, nowMs) {
|
|
3847
|
+
const budget = state.budgets[resource];
|
|
3848
|
+
if (!budget || !(unusedCost > 0)) return;
|
|
3849
|
+
const decision = resourceDecision({ budget, resource, nowMs, cost: 0, chargedCost: 0 });
|
|
3850
|
+
const callsPerMs = decision?.callsPerMs;
|
|
3851
|
+
if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) return;
|
|
3852
|
+
const credit = Math.min(unusedCost, GOVERNOR_MAX_ATOMIC_COST) / callsPerMs;
|
|
3853
|
+
budget.laneNextAt = Math.max(nowMs + HTTP_START_GAP_MS, budget.laneNextAt - credit);
|
|
3854
|
+
}
|
|
3855
|
+
|
|
2509
3856
|
function scheduleGovernorState(state, nowMs) {
|
|
2510
3857
|
if (Object.keys(state.intents).length === 0) return { grants: [], denied: [] };
|
|
2511
3858
|
let reservationCount = Object.keys(state.reservations).length;
|
|
@@ -2517,7 +3864,7 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
2517
3864
|
? [[resource, state.budgets[resource].roundRobinCursor]]
|
|
2518
3865
|
: [],
|
|
2519
3866
|
));
|
|
2520
|
-
const deferredBackground = state.
|
|
3867
|
+
const deferredBackground = state.observers.graphql.outcome === "failed"
|
|
2521
3868
|
? Object.entries(state.intents)
|
|
2522
3869
|
.filter(([, intent]) => intentPriority(intent) === REQUEST_PRIORITIES.background)
|
|
2523
3870
|
.map(([id]) => id)
|
|
@@ -2538,7 +3885,9 @@ function scheduleGovernorState(state, nowMs) {
|
|
|
2538
3885
|
cursors,
|
|
2539
3886
|
nowMs,
|
|
2540
3887
|
maxGrants: GOVERNOR_MAX_RESERVATIONS - reservationCount,
|
|
3888
|
+
manualStreak: state.fairness.manualStreak,
|
|
2541
3889
|
});
|
|
3890
|
+
state.fairness.manualStreak = result.manualStreak;
|
|
2542
3891
|
result.denied.push(...deferredBackground.map((intentId) => ({
|
|
2543
3892
|
intentId,
|
|
2544
3893
|
mode: "paused",
|
|
@@ -2625,76 +3974,79 @@ function maintainControlLease(scope, leaseId, floorMs, activeTab, nowMs) {
|
|
|
2625
3974
|
: registered;
|
|
2626
3975
|
}
|
|
2627
3976
|
|
|
2628
|
-
|
|
3977
|
+
// Claims the observer for one resource. Independence is the point: a slow or
|
|
3978
|
+
// failed GraphQL observer must not hold up core readiness, because nothing about
|
|
3979
|
+
// the resources themselves couples them. What still couples them is the shared
|
|
3980
|
+
// HTTP permit and any account-wide secondary hold, and those are enforced
|
|
3981
|
+
// elsewhere -- not by making one claim stand for both.
|
|
3982
|
+
function claimProbe(scope, leaseId, nowMs, resource) {
|
|
3983
|
+
if (!RATE_RESOURCES.includes(resource)) return { ok: false, reason: "corrupt" };
|
|
2629
3984
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2630
3985
|
if (!state.leases[leaseId]) return { ok: false, reason: "stale" };
|
|
2631
|
-
|
|
2632
|
-
|
|
3986
|
+
const held = state.probeClaims[resource];
|
|
3987
|
+
if (held && held.leaseUntil > at) {
|
|
3988
|
+
return { value: { status: "waiting", leaseUntil: held.leaseUntil } };
|
|
2633
3989
|
}
|
|
3990
|
+
const resetAt = Number.isFinite(state.budgets[resource]?.resetMs)
|
|
3991
|
+
? state.budgets[resource].resetMs + BUDGET_RESET_GRACE_MS
|
|
3992
|
+
: Number.POSITIVE_INFINITY;
|
|
3993
|
+
// A core reset starts a new shared accounting epoch, so the GraphQL counter
|
|
3994
|
+
// is also due then: external-spend baselines and the protected
|
|
3995
|
+
// one-publication reset contract have to advance together.
|
|
2634
3996
|
const coreResetAt = Number.isFinite(state.budgets.core?.resetMs)
|
|
2635
3997
|
? state.budgets.core.resetMs + BUDGET_RESET_GRACE_MS
|
|
2636
3998
|
: Number.POSITIVE_INFINITY;
|
|
2637
|
-
|
|
2638
|
-
|
|
3999
|
+
// ...but only until it has actually observed since that reset. Left
|
|
4000
|
+
// unqualified, every pane keeps qualifying for as long as core has not
|
|
4001
|
+
// published its new epoch, so a reset draws a stampede of GraphQL observers
|
|
4002
|
+
// instead of the one the shared epoch needs. The publication used to close
|
|
4003
|
+
// that window as a side effect of being read first; saying it here means it
|
|
4004
|
+
// does not depend on which resource a refresh happens to reach first.
|
|
4005
|
+
const dueAtCoreReset = resource === "graphql" &&
|
|
4006
|
+
state.observers.graphql.at < coreResetAt
|
|
4007
|
+
? coreResetAt
|
|
2639
4008
|
: Number.POSITIVE_INFINITY;
|
|
2640
|
-
const
|
|
2641
|
-
|
|
2642
|
-
|
|
2643
|
-
|
|
2644
|
-
|
|
2645
|
-
graphql: Math.min(
|
|
2646
|
-
state.probeOutcome.nextAt,
|
|
2647
|
-
graphqlResetAt,
|
|
2648
|
-
// A core reset starts a new shared accounting epoch. Refresh the free
|
|
2649
|
-
// GraphQL counter in that same nonce so external-spend baselines and
|
|
2650
|
-
// the protected one-publication reset contract advance together.
|
|
2651
|
-
coreResetAt,
|
|
2652
|
-
),
|
|
2653
|
-
};
|
|
2654
|
-
const nextAt = Math.min(...Object.values(dueAt));
|
|
4009
|
+
const nextAt = Math.min(
|
|
4010
|
+
state.observers[resource].nextAt,
|
|
4011
|
+
resetAt,
|
|
4012
|
+
dueAtCoreReset,
|
|
4013
|
+
);
|
|
2655
4014
|
if (nextAt > at) return { value: { status: "waiting", nextAt } };
|
|
2656
|
-
// One nonce owns the control-plane transition, but only the sources whose
|
|
2657
|
-
// persisted clocks are due may be read. In particular, a GraphQL minute
|
|
2658
|
-
// sample cannot spend or retry the held core observer before its reset.
|
|
2659
|
-
const resources = RATE_RESOURCES.filter((resource) => dueAt[resource] <= at);
|
|
2660
4015
|
const nonce = randomUUID();
|
|
2661
4016
|
const startedReservationIds = Object.entries(state.reservations)
|
|
2662
4017
|
// An expired owner can no longer settle its request. Keep that uncertain
|
|
2663
4018
|
// cost charged, but do not make every later probe wait for it to finish.
|
|
4019
|
+
// Only work charging this resource is drained -- an unrelated lane's
|
|
4020
|
+
// outstanding request says nothing about this counter.
|
|
2664
4021
|
.filter(([, reservation]) => reservation.status === "started" &&
|
|
2665
|
-
state.leases[reservation.leaseId]?.expiresAt > at
|
|
4022
|
+
state.leases[reservation.leaseId]?.expiresAt > at &&
|
|
4023
|
+
reservationCost(reservation, resource, state.leases, at) > 0)
|
|
2666
4024
|
.map(([id]) => id);
|
|
2667
|
-
state.
|
|
4025
|
+
state.probeClaims[resource] = {
|
|
2668
4026
|
ownerLeaseId: leaseId,
|
|
2669
4027
|
nonce,
|
|
2670
4028
|
leaseUntil: at + GOVERNOR_PROBE_LEASE_MS,
|
|
2671
4029
|
nextAt: at,
|
|
2672
4030
|
claimAt: at,
|
|
2673
4031
|
startedReservationIds,
|
|
2674
|
-
resources,
|
|
2675
4032
|
};
|
|
2676
|
-
|
|
2677
|
-
|
|
2678
|
-
|
|
2679
|
-
state.observers.core.nextAt = at + GOVERNOR_PROBE_LEASE_MS;
|
|
2680
|
-
}
|
|
2681
|
-
if (resources.includes("graphql")) {
|
|
2682
|
-
state.probeOutcome = { status: "waiting", at, nextAt: at + GOVERNOR_PROBE_LEASE_MS };
|
|
2683
|
-
}
|
|
4033
|
+
state.observers[resource].outcome = "waiting";
|
|
4034
|
+
state.observers[resource].at = at;
|
|
4035
|
+
state.observers[resource].nextAt = at + GOVERNOR_PROBE_LEASE_MS;
|
|
2684
4036
|
return { value: {
|
|
2685
4037
|
status: "claimed",
|
|
2686
4038
|
nonce,
|
|
2687
|
-
|
|
4039
|
+
resource,
|
|
4040
|
+
leaseUntil: state.probeClaims[resource].leaseUntil,
|
|
2688
4041
|
startedReservationIds,
|
|
2689
|
-
|
|
2690
|
-
coreEtag: state.observers.core.etag,
|
|
4042
|
+
coreEtag: state.observers[resource].etag,
|
|
2691
4043
|
} };
|
|
2692
4044
|
});
|
|
2693
4045
|
}
|
|
2694
4046
|
|
|
2695
|
-
function renewProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
4047
|
+
function renewProbeClaim(scope, leaseId, nonce, nowMs, resource) {
|
|
2696
4048
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2697
|
-
const claim = state.
|
|
4049
|
+
const claim = state.probeClaims[resource];
|
|
2698
4050
|
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce || claim.leaseUntil <= at) {
|
|
2699
4051
|
return { ok: false, reason: "stale" };
|
|
2700
4052
|
}
|
|
@@ -2714,8 +4066,8 @@ function budgetFromObservation(raw, previous, nowMs, {
|
|
|
2714
4066
|
if (
|
|
2715
4067
|
!normalized || normalized.resetMs <= nowMs || receivedAt > nowMs ||
|
|
2716
4068
|
!RATE_RESOURCES.includes(resource) ||
|
|
2717
|
-
!["response-header", "core-observer", "rate-limit-probe"].includes(source) ||
|
|
2718
|
-
resource === "core" &&
|
|
4069
|
+
!["response-header", "core-observer", "graphql-observer", "rate-limit-probe"].includes(source) ||
|
|
4070
|
+
resource === "core" && ["rate-limit-probe", "graphql-observer"].includes(source)
|
|
2719
4071
|
) return { status: "invalid" };
|
|
2720
4072
|
const epoch = `${normalized.limit}:${normalized.resetMs}`;
|
|
2721
4073
|
// Endpoint responses can carry cached or endpoint-specific rate headers.
|
|
@@ -2752,7 +4104,13 @@ function budgetFromObservation(raw, previous, nowMs, {
|
|
|
2752
4104
|
blockReason: blockActive ? previous.blockReason : null,
|
|
2753
4105
|
laneNextAt: epochChanged ? nowMs : previous.laneNextAt,
|
|
2754
4106
|
roundRobinCursor: previous?.roundRobinCursor ?? null,
|
|
2755
|
-
|
|
4107
|
+
// A new epoch is a new accounting window, so what other clients did in the
|
|
4108
|
+
// last one is not evidence about this one. The baseline and the local
|
|
4109
|
+
// accumulation below are already reset here; carrying the factor across
|
|
4110
|
+
// kept throttling this window on a ratio measured in a window that is over.
|
|
4111
|
+
// Only a claimed observer can open an epoch, so this cannot be moved by a
|
|
4112
|
+
// stale or reordered response.
|
|
4113
|
+
lastExternalFactor: epochChanged ? 1 : (previous?.lastExternalFactor ?? 1),
|
|
2756
4114
|
epoch,
|
|
2757
4115
|
source,
|
|
2758
4116
|
factorBaseline: epochChanged || !previous?.factorBaseline
|
|
@@ -2763,25 +4121,29 @@ function budgetFromObservation(raw, previous, nowMs, {
|
|
|
2763
4121
|
return { status: "accepted", budget: next, epochChanged };
|
|
2764
4122
|
}
|
|
2765
4123
|
|
|
2766
|
-
function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
4124
|
+
function publishProbe(scope, leaseId, nonce, budgets, nowMs, resource) {
|
|
2767
4125
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2768
|
-
const claim = state.
|
|
4126
|
+
const claim = state.probeClaims[resource];
|
|
2769
4127
|
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce || claim.leaseUntil <= at) {
|
|
2770
4128
|
return { ok: false, reason: "stale" };
|
|
2771
4129
|
}
|
|
2772
4130
|
const nextBudgets = { ...state.budgets };
|
|
2773
4131
|
const nextEpochs = { ...state.epochs };
|
|
4132
|
+
// Captured before the publication overwrites it: the reset this epoch
|
|
4133
|
+
// change supersedes is the boundary the GraphQL observer is measured against.
|
|
4134
|
+
const supersededCoreResetMs = state.budgets.core?.resetMs;
|
|
2774
4135
|
const changedResources = [];
|
|
2775
4136
|
const resetResources = [];
|
|
2776
|
-
if (!
|
|
2777
|
-
|
|
2778
|
-
|
|
2779
|
-
|
|
2780
|
-
const supplied = budgets?.[resource];
|
|
2781
|
-
if (!supplied) continue;
|
|
4137
|
+
if (!budgets?.[resource]) return { ok: false, reason: "corrupt" };
|
|
4138
|
+
const observerCosts = {};
|
|
4139
|
+
{
|
|
4140
|
+
const supplied = budgets[resource];
|
|
2782
4141
|
const raw = supplied.budget ?? supplied;
|
|
2783
|
-
|
|
2784
|
-
|
|
4142
|
+
// The owner source names what actually established the number. Calling a
|
|
4143
|
+
// GraphQL budget a "rate-limit-probe" was accurate when /rate_limit supplied
|
|
4144
|
+
// it and is a lie now that the claimed observer does.
|
|
4145
|
+
const source = supplied.source ?? (resource === "core" ? "core-observer" : "graphql-observer");
|
|
4146
|
+
const ownerSource = resource === "core" ? "core-observer" : "graphql-observer";
|
|
2785
4147
|
if (source !== ownerSource) return { ok: false, reason: "corrupt" };
|
|
2786
4148
|
const observed = budgetFromObservation(raw, state.budgets[resource], at, {
|
|
2787
4149
|
resource,
|
|
@@ -2791,15 +4153,20 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
|
2791
4153
|
allowEpochChange: true,
|
|
2792
4154
|
});
|
|
2793
4155
|
if (observed.status === "invalid") return { ok: false, reason: "corrupt" };
|
|
2794
|
-
if (observed.status
|
|
2795
|
-
|
|
2796
|
-
|
|
2797
|
-
|
|
2798
|
-
|
|
4156
|
+
if (observed.status !== "ignored") {
|
|
4157
|
+
// The observer's own cost is local spend. Left out, the external-factor
|
|
4158
|
+
// reconciliation below attributes it to other clients, and the governor
|
|
4159
|
+
// throttles the user's real work to make room for its own probing.
|
|
4160
|
+
observerCosts[resource] = Number.isSafeInteger(supplied.cost) && supplied.cost > 0 ? supplied.cost : 0;
|
|
4161
|
+
nextBudgets[resource] = observed.budget;
|
|
4162
|
+
if (resource === "core" && supplied.blocked === true) {
|
|
4163
|
+
nextBudgets[resource].blockUntil = observed.budget.resetMs;
|
|
4164
|
+
nextBudgets[resource].blockReason = "rate-limit";
|
|
4165
|
+
}
|
|
4166
|
+
nextEpochs[resource] = observed.budget.epoch;
|
|
4167
|
+
changedResources.push(resource);
|
|
4168
|
+
if (observed.epochChanged && state.epochs[resource] !== null) resetResources.push(resource);
|
|
2799
4169
|
}
|
|
2800
|
-
nextEpochs[resource] = observed.budget.epoch;
|
|
2801
|
-
changedResources.push(resource);
|
|
2802
|
-
if (observed.epochChanged && state.epochs[resource] !== null) resetResources.push(resource);
|
|
2803
4170
|
}
|
|
2804
4171
|
if (changedResources.length === 0) return { ok: false, reason: "corrupt" };
|
|
2805
4172
|
const completedBeforeClaim = Object.entries(state.reservations)
|
|
@@ -2817,19 +4184,32 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
|
2817
4184
|
total + (reservation.completedAt > baseline.observedAt
|
|
2818
4185
|
? reservationCost(reservation, resource, state.leases, at)
|
|
2819
4186
|
: 0), 0);
|
|
4187
|
+
const globalUsedDelta = nextBudgets[resource].used - baseline.used;
|
|
4188
|
+
const sharedCompletedDelta = previous.knownLocalUsed + legacyCompletedDelta + (observerCosts[resource] ?? 0);
|
|
2820
4189
|
const factor = nextExternalFactor({
|
|
2821
4190
|
lastExternalFactor: previous.lastExternalFactor,
|
|
2822
|
-
globalUsedDelta
|
|
2823
|
-
sharedCompletedDelta
|
|
4191
|
+
globalUsedDelta,
|
|
4192
|
+
sharedCompletedDelta,
|
|
2824
4193
|
});
|
|
2825
4194
|
if (factor === null) return { ok: false, reason: "corrupt" };
|
|
2826
4195
|
nextBudgets[resource].lastExternalFactor = factor;
|
|
2827
|
-
|
|
2828
|
-
|
|
2829
|
-
|
|
2830
|
-
|
|
2831
|
-
|
|
2832
|
-
|
|
4196
|
+
// Closing the window costs the evidence in it, so only a window that
|
|
4197
|
+
// actually reconciled may close. Four local units observed twice are
|
|
4198
|
+
// eight, and eight are enough; discarding each four because neither
|
|
4199
|
+
// reached five on its own is what kept a factor of seven alive through
|
|
4200
|
+
// any amount of purely local spend.
|
|
4201
|
+
if (externalSampleIsUsable({ globalUsedDelta, sharedCompletedDelta })) {
|
|
4202
|
+
nextBudgets[resource].factorBaseline = {
|
|
4203
|
+
epoch: nextBudgets[resource].epoch,
|
|
4204
|
+
used: nextBudgets[resource].used,
|
|
4205
|
+
observedAt: nextBudgets[resource].observedAt,
|
|
4206
|
+
};
|
|
4207
|
+
nextBudgets[resource].knownLocalUsed = 0;
|
|
4208
|
+
}
|
|
4209
|
+
// Otherwise both are left exactly as the observation produced them, which
|
|
4210
|
+
// is the previous baseline and the previous local total. Rewriting them
|
|
4211
|
+
// here would count the same completed reservations twice: they are still
|
|
4212
|
+
// in the ledger precisely because the baseline did not advance past them.
|
|
2833
4213
|
}
|
|
2834
4214
|
for (const [id, reservation] of completedBeforeClaim) {
|
|
2835
4215
|
const accountedByEveryResource = RATE_RESOURCES.every((resource) =>
|
|
@@ -2839,24 +4219,41 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
|
2839
4219
|
}
|
|
2840
4220
|
state.budgets = nextBudgets;
|
|
2841
4221
|
state.epochs = nextEpochs;
|
|
2842
|
-
state.
|
|
2843
|
-
|
|
2844
|
-
const
|
|
2845
|
-
state.observers
|
|
2846
|
-
etag: typeof budgets.
|
|
4222
|
+
state.probeClaims[resource] = null;
|
|
4223
|
+
{
|
|
4224
|
+
const published = nextBudgets[resource];
|
|
4225
|
+
state.observers[resource] = {
|
|
4226
|
+
etag: typeof budgets[resource].etag === "string" ? budgets[resource].etag : state.observers[resource].etag,
|
|
2847
4227
|
outcome: "healthy",
|
|
2848
4228
|
at,
|
|
2849
|
-
|
|
2850
|
-
|
|
4229
|
+
// A resource observed as empty is due again at its own reset, not at the
|
|
4230
|
+
// ordinary cadence: probing an exhausted counter to be told it is still
|
|
4231
|
+
// exhausted spends against it for nothing.
|
|
4232
|
+
nextAt: published.remaining === 0
|
|
4233
|
+
? published.resetMs + BUDGET_RESET_GRACE_MS
|
|
2851
4234
|
: at + BUDGET_PROBE_MS,
|
|
2852
4235
|
};
|
|
2853
4236
|
}
|
|
2854
|
-
|
|
2855
|
-
|
|
2856
|
-
|
|
2857
|
-
|
|
2858
|
-
|
|
2859
|
-
|
|
4237
|
+
// A core reset opens a new shared accounting epoch, so the GraphQL counter
|
|
4238
|
+
// is due with it: external-spend baselines and the protected
|
|
4239
|
+
// one-publication reset contract have to advance together. A refresh that
|
|
4240
|
+
// reads GraphQL before core has already done this, because core's old reset
|
|
4241
|
+
// time still said GraphQL was due when that claim was evaluated. What
|
|
4242
|
+
// cannot see it is a pane whose GraphQL observer last ran before the reset
|
|
4243
|
+
// -- another pane published this epoch first, moving the reset out of
|
|
4244
|
+
// reach. Only that case still owes an observation, so only that case is
|
|
4245
|
+
// pulled forward; marking it unconditionally spends a second point to
|
|
4246
|
+
// re-read a counter this same cycle has just read. An exhausted GraphQL
|
|
4247
|
+
// counter waits for its own reset either way.
|
|
4248
|
+
if (resource === "core" && resetResources.includes("core")) {
|
|
4249
|
+
const graphql = nextBudgets.graphql;
|
|
4250
|
+
const heldToReset = graphql && graphql.remaining === 0 &&
|
|
4251
|
+
at < graphql.resetMs + BUDGET_RESET_GRACE_MS;
|
|
4252
|
+
const observedSinceReset = Number.isFinite(supersededCoreResetMs) &&
|
|
4253
|
+
state.observers.graphql.at >= supersededCoreResetMs;
|
|
4254
|
+
if (!heldToReset && !observedSinceReset) {
|
|
4255
|
+
state.observers.graphql.nextAt = Math.min(state.observers.graphql.nextAt || at, at);
|
|
4256
|
+
}
|
|
2860
4257
|
}
|
|
2861
4258
|
if (resetResources.length > 0) state.manualProbe = null;
|
|
2862
4259
|
else if (state.manualProbe &&
|
|
@@ -2873,24 +4270,22 @@ function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
|
2873
4270
|
});
|
|
2874
4271
|
}
|
|
2875
4272
|
|
|
2876
|
-
function failProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
4273
|
+
function failProbeClaim(scope, leaseId, nonce, nowMs, resource) {
|
|
2877
4274
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2878
|
-
const claim = state.
|
|
4275
|
+
const claim = state.probeClaims[resource];
|
|
2879
4276
|
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce) {
|
|
2880
4277
|
return { ok: false, reason: "stale" };
|
|
2881
4278
|
}
|
|
2882
|
-
|
|
2883
|
-
|
|
2884
|
-
|
|
2885
|
-
|
|
2886
|
-
|
|
2887
|
-
state.observers
|
|
2888
|
-
|
|
2889
|
-
|
|
2890
|
-
|
|
2891
|
-
|
|
2892
|
-
};
|
|
2893
|
-
}
|
|
4279
|
+
// Only this resource's readiness is affected. A failed GraphQL observer that
|
|
4280
|
+
// also marked core failed would erase authority core had legitimately
|
|
4281
|
+
// established, and stall a lane with nothing wrong with it.
|
|
4282
|
+
state.probeClaims[resource] = null;
|
|
4283
|
+
state.observers[resource] = {
|
|
4284
|
+
...state.observers[resource],
|
|
4285
|
+
outcome: "failed",
|
|
4286
|
+
at,
|
|
4287
|
+
nextAt: at + BUDGET_PROBE_MS,
|
|
4288
|
+
};
|
|
2894
4289
|
return { value: { retryAt: at + BUDGET_PROBE_MS } };
|
|
2895
4290
|
});
|
|
2896
4291
|
}
|
|
@@ -2912,7 +4307,7 @@ function requestManualProbe(scope, leaseId, epoch, observedAt, nowMs) {
|
|
|
2912
4307
|
: Math.min(state.observers.core.nextAt || at, at);
|
|
2913
4308
|
}
|
|
2914
4309
|
if (state.epochs.graphql === epoch) {
|
|
2915
|
-
state.
|
|
4310
|
+
state.observers.graphql.nextAt = Math.min(state.observers.graphql.nextAt || at, at);
|
|
2916
4311
|
}
|
|
2917
4312
|
return { value: { status: "pending", ...state.manualProbe } };
|
|
2918
4313
|
});
|
|
@@ -2970,16 +4365,23 @@ function registerIntent(scope, intent) {
|
|
|
2970
4365
|
});
|
|
2971
4366
|
}
|
|
2972
4367
|
|
|
2973
|
-
function readIntentDecision(scope, intentId, nowMs) {
|
|
4368
|
+
function readIntentDecision(scope, intentId, nowMs, previousEvidence = null) {
|
|
2974
4369
|
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2975
4370
|
const scheduled = scheduleGovernorState(state, at);
|
|
2976
4371
|
const reservationId = `reservation:${intentId}`;
|
|
2977
4372
|
const reservation = state.reservations[reservationId];
|
|
2978
4373
|
const grant = scheduled.grants.find((item) => item.id === reservationId);
|
|
4374
|
+
// UI provenance stays process-local. Reuse it only for this same pending
|
|
4375
|
+
// intent while its original owners and pane count remain valid.
|
|
4376
|
+
const retainedSharing = previousEvidence?.intentId === intentId &&
|
|
4377
|
+
reservation?.status === "scheduled" && reservation.notBefore > at &&
|
|
4378
|
+
currentSharedLaneProvenance(previousEvidence, state.leases, at).waitCause
|
|
4379
|
+
? sharedLaneEvidence(previousEvidence) : {};
|
|
2979
4380
|
if (reservation) return { value: {
|
|
2980
4381
|
status: reservation.status,
|
|
2981
4382
|
reservationId,
|
|
2982
4383
|
...reservation,
|
|
4384
|
+
...retainedSharing,
|
|
2983
4385
|
...sharedLaneEvidence(grant),
|
|
2984
4386
|
} };
|
|
2985
4387
|
if (!state.intents[intentId]) return { ok: false, reason: "stale" };
|
|
@@ -3008,6 +4410,10 @@ function cancelIntent(scope, intentId, nowMs) {
|
|
|
3008
4410
|
return { ok: false, reason: "stale" };
|
|
3009
4411
|
}
|
|
3010
4412
|
delete state.reservations[reservationId];
|
|
4413
|
+
// The request never started, so the whole slot it was paced into is free.
|
|
4414
|
+
for (const resource of RATE_RESOURCES) {
|
|
4415
|
+
returnPacingCredit(state, resource, reservation.costs[resource], nowMs ?? Date.now());
|
|
4416
|
+
}
|
|
3011
4417
|
return { value: { status: "cancelled", intentId, reservationId } };
|
|
3012
4418
|
});
|
|
3013
4419
|
}
|
|
@@ -3021,8 +4427,14 @@ function startReservation(scope, reservationId, nowMs) {
|
|
|
3021
4427
|
return { ok: false, reason: "stale", write: Boolean(reservation) };
|
|
3022
4428
|
}
|
|
3023
4429
|
if (reservation.notBefore > at) return { value: { status: "waiting", notBefore: reservation.notBefore } };
|
|
3024
|
-
|
|
3025
|
-
|
|
4430
|
+
// Any live observer claim defers a start: the claim owns the shared HTTP
|
|
4431
|
+
// permit for its window, so starting here would contend for it.
|
|
4432
|
+
const liveClaim = RATE_RESOURCES
|
|
4433
|
+
.map((resource) => state.probeClaims[resource])
|
|
4434
|
+
.filter((claim) => claim && claim.leaseUntil > at)
|
|
4435
|
+
.sort((left, right) => right.leaseUntil - left.leaseUntil)[0];
|
|
4436
|
+
if (liveClaim) {
|
|
4437
|
+
return { value: { status: "waiting", reason: "probe", notBefore: liveClaim.leaseUntil } };
|
|
3026
4438
|
}
|
|
3027
4439
|
for (const resource of RATE_RESOURCES.filter((name) => reservation.costs[name] > 0)) {
|
|
3028
4440
|
const budget = state.budgets[resource];
|
|
@@ -3157,6 +4569,15 @@ function settleReservationWithBudgetObservations(
|
|
|
3157
4569
|
acceptedAfterBaseline[resource],
|
|
3158
4570
|
);
|
|
3159
4571
|
}
|
|
4572
|
+
// Only a measured outcome knows what it spent. A timeout, abort or process
|
|
4573
|
+
// loss proves nothing, so its worst case stays charged and its pacing stays
|
|
4574
|
+
// spent -- refunding there would let a run of timeouts pace as though
|
|
4575
|
+
// nothing had been sent.
|
|
4576
|
+
if (completion.outcome === "measured-success") {
|
|
4577
|
+
for (const resource of RATE_RESOURCES) {
|
|
4578
|
+
returnPacingCredit(state, resource, reservation.costs[resource] - measured[resource], at);
|
|
4579
|
+
}
|
|
4580
|
+
}
|
|
3160
4581
|
scheduleGovernorState(state, at);
|
|
3161
4582
|
return { value: {
|
|
3162
4583
|
status: "completed",
|
|
@@ -3179,8 +4600,8 @@ function recordResourceBlock(scope, resource, resetMs, reason) {
|
|
|
3179
4600
|
resetMs + BUDGET_RESET_GRACE_MS,
|
|
3180
4601
|
);
|
|
3181
4602
|
} else {
|
|
3182
|
-
state.
|
|
3183
|
-
state.
|
|
4603
|
+
state.observers.graphql.nextAt = Math.min(
|
|
4604
|
+
state.observers.graphql.nextAt || resetMs + BUDGET_RESET_GRACE_MS,
|
|
3184
4605
|
resetMs + BUDGET_RESET_GRACE_MS,
|
|
3185
4606
|
);
|
|
3186
4607
|
}
|
|
@@ -3216,7 +4637,7 @@ function governorHealth(result, nowMs = Date.now()) {
|
|
|
3216
4637
|
if (!result?.ok) return { status: "unavailable", leases: 0, resources: {} };
|
|
3217
4638
|
const state = result.value;
|
|
3218
4639
|
let status = "healthy";
|
|
3219
|
-
if (state.
|
|
4640
|
+
if (RATE_RESOURCES.some((resource) => state.probeClaims[resource]?.leaseUntil > nowMs)) status = "waiting for probe";
|
|
3220
4641
|
else if (RATE_RESOURCES.some((resource) => !state.budgets[resource] ||
|
|
3221
4642
|
nowMs - state.budgets[resource].observedAt > budgetSnapshotTtl(resource))) status = "stale";
|
|
3222
4643
|
else if (RATE_RESOURCES.some((resource) => state.budgets[resource].blockUntil > nowMs)) status = "blocked";
|
|
@@ -3244,7 +4665,40 @@ async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
|
|
|
3244
4665
|
return result;
|
|
3245
4666
|
}
|
|
3246
4667
|
|
|
3247
|
-
|
|
4668
|
+
// Each resource refreshes on its own claim. A slow or failing observer for one
|
|
4669
|
+
// must not delay or invalidate the other's readiness -- nothing about the
|
|
4670
|
+
// resources couples them. What still couples them is the shared HTTP permit and
|
|
4671
|
+
// any account-wide secondary hold, and those are enforced where they belong.
|
|
4672
|
+
// Core is refreshed last because its publication is what reopens data
|
|
4673
|
+
// admission. Finishing the cycle's control work before that happens stops a
|
|
4674
|
+
// pane that merely published the reset from getting a head start, over the
|
|
4675
|
+
// shared permit, on higher-priority work waiting for the same lane. It is also
|
|
4676
|
+
// the order readSharedBudgetSources reads its sources in, and the order the
|
|
4677
|
+
// single claim this replaced already used, so the sequence of actual HTTP
|
|
4678
|
+
// calls is unchanged.
|
|
4679
|
+
const BUDGET_REFRESH_ORDER = ["graphql", "core"];
|
|
4680
|
+
|
|
4681
|
+
async function refreshSharedBudget(scope, leaseId, signal, options = {}) {
|
|
4682
|
+
const outcomes = [];
|
|
4683
|
+
for (const resource of BUDGET_REFRESH_ORDER) {
|
|
4684
|
+
outcomes.push([resource, await refreshResourceBudget(scope, leaseId, signal, resource, options)]);
|
|
4685
|
+
}
|
|
4686
|
+
const published = outcomes.filter(([, result]) => result.ok);
|
|
4687
|
+
if (published.length === 0) return outcomes[0][1];
|
|
4688
|
+
const inspect = options.inspect ?? inspectGovernor;
|
|
4689
|
+
const now = options.now ?? (() => scopeNow(scope));
|
|
4690
|
+
const snapshot = inspect(scope, now());
|
|
4691
|
+
return {
|
|
4692
|
+
ok: true,
|
|
4693
|
+
value: {
|
|
4694
|
+
status: "published",
|
|
4695
|
+
budgets: snapshot.ok ? snapshot.value.budgets : {},
|
|
4696
|
+
resources: published.map(([resource]) => resource),
|
|
4697
|
+
},
|
|
4698
|
+
};
|
|
4699
|
+
}
|
|
4700
|
+
|
|
4701
|
+
async function refreshResourceBudget(scope, leaseId, signal, resource, {
|
|
3248
4702
|
readBudgets = readSharedBudgetSources,
|
|
3249
4703
|
now = () => scopeNow(scope),
|
|
3250
4704
|
wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
@@ -3253,7 +4707,7 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
3253
4707
|
publish = publishProbe,
|
|
3254
4708
|
fail = failProbeClaim,
|
|
3255
4709
|
} = {}) {
|
|
3256
|
-
let claim = claimProbe(scope, leaseId, now());
|
|
4710
|
+
let claim = claimProbe(scope, leaseId, now(), resource);
|
|
3257
4711
|
if (!claim.ok) return claim;
|
|
3258
4712
|
if (claim.value.status !== "claimed") {
|
|
3259
4713
|
const startedAt = now();
|
|
@@ -3266,32 +4720,33 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
3266
4720
|
inspections += 1;
|
|
3267
4721
|
const snapshot = inspect(scope, now());
|
|
3268
4722
|
if (!snapshot.ok) return snapshot;
|
|
4723
|
+
// Only this resource's freshness matters here. Waiting on the other's
|
|
4724
|
+
// observer is what made one failing lane stall an unrelated healthy one.
|
|
3269
4725
|
if (
|
|
3270
|
-
|
|
3271
|
-
|
|
3272
|
-
|
|
3273
|
-
) && snapshot.value.probeOutcome.status === "healthy"
|
|
4726
|
+
snapshot.value.budgets[resource] &&
|
|
4727
|
+
now() - snapshot.value.budgets[resource].observedAt <= budgetSnapshotTtl(resource) &&
|
|
4728
|
+
snapshot.value.observers[resource].outcome === "healthy"
|
|
3274
4729
|
) {
|
|
3275
4730
|
return { ok: true, value: { status: "published", budgets: snapshot.value.budgets } };
|
|
3276
4731
|
}
|
|
3277
4732
|
if (now() >= deadline || inspections >= 10) break;
|
|
3278
4733
|
await wait(Math.min(100, Math.max(1, deadline - now())));
|
|
3279
4734
|
}
|
|
3280
|
-
claim = claimProbe(scope, leaseId, now());
|
|
4735
|
+
claim = claimProbe(scope, leaseId, now(), resource);
|
|
3281
4736
|
if (!claim.ok || claim.value.status !== "claimed") return claim;
|
|
3282
4737
|
}
|
|
3283
|
-
const { nonce, startedReservationIds,
|
|
4738
|
+
const { nonce, startedReservationIds, coreEtag = null } = claim.value;
|
|
3284
4739
|
const drainUntil = now() + GOVERNOR_PROBE_DRAIN_MS;
|
|
3285
4740
|
while (startedReservationIds.length > 0 && now() < drainUntil) {
|
|
3286
4741
|
const snapshot = inspect(scope, now());
|
|
3287
4742
|
if (!snapshot.ok) {
|
|
3288
|
-
failProbeClaim(scope, leaseId, nonce, now());
|
|
4743
|
+
failProbeClaim(scope, leaseId, nonce, now(), resource);
|
|
3289
4744
|
return snapshot;
|
|
3290
4745
|
}
|
|
3291
4746
|
const stillStarted = startedReservationIds.some((id) => snapshot.value.reservations[id]?.status === "started");
|
|
3292
4747
|
if (!stillStarted) break;
|
|
3293
4748
|
if (signal?.aborted) {
|
|
3294
|
-
failProbeClaim(scope, leaseId, nonce, now());
|
|
4749
|
+
failProbeClaim(scope, leaseId, nonce, now(), resource);
|
|
3295
4750
|
return { ok: false, reason: "stale" };
|
|
3296
4751
|
}
|
|
3297
4752
|
await wait(Math.min(100, Math.max(1, drainUntil - now())));
|
|
@@ -3302,22 +4757,35 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
3302
4757
|
renewalStartedAt + GOVERNOR_PROBE_TRANSITION_MS,
|
|
3303
4758
|
);
|
|
3304
4759
|
const renewed = await retryGovernorMutation(
|
|
3305
|
-
() => renew(scope, leaseId, nonce, now()),
|
|
4760
|
+
() => renew(scope, leaseId, nonce, now(), resource),
|
|
3306
4761
|
{ now, wait, deadline: renewalDeadline, signal },
|
|
3307
4762
|
);
|
|
3308
4763
|
if (!renewed.ok) {
|
|
3309
|
-
failProbeClaim(scope, leaseId, nonce, now());
|
|
4764
|
+
failProbeClaim(scope, leaseId, nonce, now(), resource);
|
|
3310
4765
|
return renewed;
|
|
3311
4766
|
}
|
|
4767
|
+
// Known exhaustion waits for its actual reset. The GraphQL observer costs a
|
|
4768
|
+
// point, so probing an already-empty budget to be told it is still empty digs
|
|
4769
|
+
// the hole deeper -- the /rate_limit read this replaced was free and could
|
|
4770
|
+
// afford to be unconditional.
|
|
4771
|
+
const beforeRead = inspect(scope, now());
|
|
4772
|
+
const exhausted = resource === "graphql" && beforeRead.ok &&
|
|
4773
|
+
beforeRead.value.budgets.graphql &&
|
|
4774
|
+
beforeRead.value.budgets.graphql.remaining <= 0 &&
|
|
4775
|
+
now() < beforeRead.value.budgets.graphql.resetMs + BUDGET_RESET_GRACE_MS;
|
|
4776
|
+
if (exhausted) {
|
|
4777
|
+
failProbeClaim(scope, leaseId, nonce, now(), resource);
|
|
4778
|
+
return { ok: false, reason: "budget-reset" };
|
|
4779
|
+
}
|
|
3312
4780
|
let budgets;
|
|
3313
4781
|
try {
|
|
3314
|
-
budgets = await readBudgets(signal, scope.host, {
|
|
3315
|
-
resources,
|
|
4782
|
+
budgets = await requestIdentityStorage.run(scope, () => readBudgets(signal, scope.host, {
|
|
4783
|
+
resources: [resource],
|
|
3316
4784
|
coreEtag,
|
|
3317
4785
|
renewClaim: async () => {
|
|
3318
4786
|
const startedAt = now();
|
|
3319
4787
|
const extended = await retryGovernorMutation(
|
|
3320
|
-
() => renew(scope, leaseId, nonce, now()),
|
|
4788
|
+
() => renew(scope, leaseId, nonce, now(), resource),
|
|
3321
4789
|
{
|
|
3322
4790
|
now,
|
|
3323
4791
|
wait,
|
|
@@ -3328,7 +4796,7 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
3328
4796
|
if (extended.ok) renewed.value.leaseUntil = extended.value.leaseUntil;
|
|
3329
4797
|
return extended.ok;
|
|
3330
4798
|
},
|
|
3331
|
-
});
|
|
4799
|
+
}));
|
|
3332
4800
|
} catch {
|
|
3333
4801
|
budgets = null;
|
|
3334
4802
|
}
|
|
@@ -3338,7 +4806,7 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
3338
4806
|
now() + GOVERNOR_PROBE_TRANSITION_MS,
|
|
3339
4807
|
);
|
|
3340
4808
|
await retryGovernorMutation(
|
|
3341
|
-
() => fail(scope, leaseId, nonce, now()),
|
|
4809
|
+
() => fail(scope, leaseId, nonce, now(), resource),
|
|
3342
4810
|
{ now, wait, deadline: failureDeadline, signal },
|
|
3343
4811
|
);
|
|
3344
4812
|
return { ok: false, reason: "stale" };
|
|
@@ -3348,12 +4816,12 @@ async function refreshSharedBudget(scope, leaseId, signal, {
|
|
|
3348
4816
|
now() + GOVERNOR_PROBE_TRANSITION_MS,
|
|
3349
4817
|
);
|
|
3350
4818
|
const published = await retryGovernorMutation(
|
|
3351
|
-
() => publish(scope, leaseId, nonce, budgets, now()),
|
|
4819
|
+
() => publish(scope, leaseId, nonce, budgets, now(), resource),
|
|
3352
4820
|
{ now, wait, deadline: transitionDeadline, signal },
|
|
3353
4821
|
);
|
|
3354
4822
|
if (!published.ok) {
|
|
3355
4823
|
await retryGovernorMutation(
|
|
3356
|
-
() => fail(scope, leaseId, nonce, now()),
|
|
4824
|
+
() => fail(scope, leaseId, nonce, now(), resource),
|
|
3357
4825
|
{ now, wait, deadline: transitionDeadline, signal },
|
|
3358
4826
|
);
|
|
3359
4827
|
}
|
|
@@ -3417,8 +4885,13 @@ function mergeAlertRows(groups) {
|
|
|
3417
4885
|
// mid-session is picked up within the hour.
|
|
3418
4886
|
const alertBackoff = new Map();
|
|
3419
4887
|
|
|
4888
|
+
function backoffStorageKey(key) {
|
|
4889
|
+
const access = requestIdentityStorage.getStore()?.accessKey ?? runtimeIdentityCoordinator?.current()?.accessKey;
|
|
4890
|
+
return access ? `${access}\0${key}` : key;
|
|
4891
|
+
}
|
|
4892
|
+
|
|
3420
4893
|
function backoffActive(key, now) {
|
|
3421
|
-
const state = alertBackoff.get(key);
|
|
4894
|
+
const state = alertBackoff.get(backoffStorageKey(key));
|
|
3422
4895
|
return Boolean(state) && now < state.until;
|
|
3423
4896
|
}
|
|
3424
4897
|
|
|
@@ -3434,13 +4907,13 @@ const FAILURE_LADDER = {
|
|
|
3434
4907
|
// which is the drift the ALERT_SOURCES comment below warns about: two copies of
|
|
3435
4908
|
// this would diverge the first time one of them was fixed.
|
|
3436
4909
|
function recordFailure(key, now, steps = BACKOFF_STEPS_MS) {
|
|
3437
|
-
const previous = alertBackoff.get(key);
|
|
4910
|
+
const previous = alertBackoff.get(backoffStorageKey(key));
|
|
3438
4911
|
const step = Math.min((previous?.step ?? -1) + 1, steps.length - 1);
|
|
3439
|
-
alertBackoff.set(key, { step, until: now + steps[step] });
|
|
4912
|
+
alertBackoff.set(backoffStorageKey(key), { step, until: now + steps[step] });
|
|
3440
4913
|
}
|
|
3441
4914
|
|
|
3442
4915
|
function clearBackoff(key) {
|
|
3443
|
-
alertBackoff.delete(key);
|
|
4916
|
+
alertBackoff.delete(backoffStorageKey(key));
|
|
3444
4917
|
}
|
|
3445
4918
|
|
|
3446
4919
|
function forcedBackoffKeys(key) {
|
|
@@ -3462,7 +4935,7 @@ async function fetchAlertSource(source, signal, now, {
|
|
|
3462
4935
|
request = fetchConditionalEntity,
|
|
3463
4936
|
} = {}) {
|
|
3464
4937
|
if (backoffActive(source.key, now)) {
|
|
3465
|
-
const { note, verdict } = alertBackoff.get(source.key);
|
|
4938
|
+
const { note, verdict } = alertBackoff.get(backoffStorageKey(source.key));
|
|
3466
4939
|
// The verdict is replayed alongside the note. Replaying only the note left
|
|
3467
4940
|
// the tab unable to tell "Dependabot is switched off here" from "we cannot
|
|
3468
4941
|
// see Dependabot" for the whole length of a backoff window.
|
|
@@ -3557,7 +5030,7 @@ async function fetchAlertSource(source, signal, now, {
|
|
|
3557
5030
|
const steps = pick(FAILURE_LADDER, verdict, null);
|
|
3558
5031
|
if (steps) {
|
|
3559
5032
|
recordFailure(source.key, now, steps);
|
|
3560
|
-
Object.assign(alertBackoff.get(source.key), { note, verdict });
|
|
5033
|
+
Object.assign(alertBackoff.get(backoffStorageKey(source.key)), { note, verdict });
|
|
3561
5034
|
}
|
|
3562
5035
|
return {
|
|
3563
5036
|
raw: `unavailable:${note}`,
|
|
@@ -3668,36 +5141,72 @@ function reconcileSelectionViewport({ items, key, offset = 0, rows = 1 }) {
|
|
|
3668
5141
|
return { key, offset: nextOffset };
|
|
3669
5142
|
}
|
|
3670
5143
|
|
|
3671
|
-
//
|
|
3672
|
-
//
|
|
3673
|
-
|
|
3674
|
-
|
|
5144
|
+
// The tabs with a per-item URL. Security is absent on purpose: an alert has no
|
|
5145
|
+
// stable per-item page this app can construct honestly.
|
|
5146
|
+
const OPENABLE = ["actions", "issues", "prs"];
|
|
5147
|
+
|
|
5148
|
+
// The platform's opener, by fixed argv and without a shell, so a URL is an
|
|
5149
|
+
// argument and never a fragment of a command line.
|
|
5150
|
+
// rundll32 rather than `cmd /c start ""`: cmd re-parses its /c string, so a URL
|
|
5151
|
+
// handed to it is a fragment of a command line and its metacharacters are live.
|
|
5152
|
+
// FileProtocolHandler takes the URL as one real argument, which is the whole
|
|
5153
|
+
// point of using execFile without a shell.
|
|
5154
|
+
const BROWSER_OPENERS = {
|
|
5155
|
+
darwin: ["open"],
|
|
5156
|
+
linux: ["xdg-open"],
|
|
5157
|
+
win32: ["rundll32", "url.dll,FileProtocolHandler"],
|
|
5158
|
+
};
|
|
3675
5159
|
|
|
3676
|
-
//
|
|
3677
|
-
//
|
|
3678
|
-
//
|
|
3679
|
-
|
|
3680
|
-
|
|
3681
|
-
|
|
3682
|
-
|
|
3683
|
-
|
|
3684
|
-
|
|
3685
|
-
|
|
3686
|
-
|
|
3687
|
-
|
|
3688
|
-
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
3696
|
-
|
|
3697
|
-
|
|
3698
|
-
|
|
5160
|
+
// Only https, and only the host this pane is actually talking to. A row is
|
|
5161
|
+
// remote data; without this a crafted `url` could point the user's browser
|
|
5162
|
+
// anywhere, or hand a `file:`/`javascript:` URL to the platform opener.
|
|
5163
|
+
function admittedRowUrl(value, host) {
|
|
5164
|
+
if (typeof value !== "string" || value.length === 0) return null;
|
|
5165
|
+
let parsed;
|
|
5166
|
+
try { parsed = new URL(value); } catch { return null; }
|
|
5167
|
+
if (parsed.protocol !== "https:") return null;
|
|
5168
|
+
return parsed.host === host ? parsed.toString() : null;
|
|
5169
|
+
}
|
|
5170
|
+
|
|
5171
|
+
// Actions rows come from REST and carry no URL, but a run's page is a pure
|
|
5172
|
+
// function of the repository and its databaseId -- both already validated --
|
|
5173
|
+
// so it is derived locally rather than fetched. Issues and PRs select `url` in
|
|
5174
|
+
// their query, which is validated against the same host before use.
|
|
5175
|
+
function rowBrowserUrl(tabKey, item, { host = effectiveRuntimeHost(), repo = effectiveRuntimeRepository() } = {}) {
|
|
5176
|
+
if (!OPENABLE.includes(tabKey) || !isRecord(item) || !host) return null;
|
|
5177
|
+
// The row's own URL first, for every tab. A run only falls back to a derived
|
|
5178
|
+
// URL when the payload predates the projection, and that fallback needs the
|
|
5179
|
+
// slug, which the app only has when --repo was given.
|
|
5180
|
+
const declared = admittedRowUrl(item.url, host);
|
|
5181
|
+
if (declared || tabKey !== "actions") return declared;
|
|
5182
|
+
const id = item.databaseId ?? item.number;
|
|
5183
|
+
if (!Number.isSafeInteger(id) || id <= 0 || !/^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/.test(String(repo ?? ""))) return null;
|
|
5184
|
+
return `https://${host}/${repo}/actions/runs/${id}`;
|
|
5185
|
+
}
|
|
5186
|
+
|
|
5187
|
+
// Opening a page the row already names costs nothing and needs no admission:
|
|
5188
|
+
// `gh <kind> view --web` spent a request to be told a URL that was already in
|
|
5189
|
+
// hand. Output is still captured rather than inherited, because stdout is ink's
|
|
5190
|
+
// frame stream and an opener's chatter would corrupt the alternate screen.
|
|
5191
|
+
async function openInBrowser(tabKey, item, signal, _governor = null, { run = null, host, repo } = {}) {
|
|
5192
|
+
if (!OPENABLE.includes(tabKey)) return;
|
|
5193
|
+
const url = rowBrowserUrl(tabKey, item, {
|
|
5194
|
+
...(host === undefined ? {} : { host }),
|
|
5195
|
+
...(repo === undefined ? {} : { repo }),
|
|
3699
5196
|
});
|
|
3700
|
-
|
|
5197
|
+
// Returning quietly here is how a dead Enter key looks: the row has no usable
|
|
5198
|
+
// page, and the user is entitled to know that rather than press it again.
|
|
5199
|
+
if (!url) throw new Error("This row has no page to open yet; refresh and try again");
|
|
5200
|
+
const opener = pick(BROWSER_OPENERS, process.platform, null);
|
|
5201
|
+
if (!opener) throw new Error(`Opening a browser is not supported on ${process.platform}`);
|
|
5202
|
+
const [command, ...prefix] = opener;
|
|
5203
|
+
const launch = run ?? ((argv) => execFileAsync(argv[0], argv.slice(1), {
|
|
5204
|
+
timeout: GH_TIMEOUT_MS,
|
|
5205
|
+
killSignal: "SIGKILL",
|
|
5206
|
+
maxBuffer: GH_MAX_BUFFER,
|
|
5207
|
+
signal,
|
|
5208
|
+
}));
|
|
5209
|
+
await launch([command, ...prefix, url]);
|
|
3701
5210
|
}
|
|
3702
5211
|
|
|
3703
5212
|
function createOpenRequestRegistry() {
|
|
@@ -3863,10 +5372,16 @@ async function captureGh(args, operation) {
|
|
|
3863
5372
|
|
|
3864
5373
|
const PROBE_STDERR_LIMIT = 400;
|
|
3865
5374
|
|
|
3866
|
-
async function probe(name, args, operation) {
|
|
5375
|
+
async function probe(name, args, operation, input = null) {
|
|
3867
5376
|
const startedAt = Date.now();
|
|
5377
|
+
// A declared local prerequisite the run cannot satisfy is reported, not
|
|
5378
|
+
// attempted: starting a request whose document cannot be built would spend a
|
|
5379
|
+
// reservation to receive a GraphQL error that says nothing about the tenant.
|
|
5380
|
+
if (input === UNRESOLVED_REPOSITORY) {
|
|
5381
|
+
return { name, args, ms: 0, failed: true, stderr: UNRESOLVED_REPOSITORY, http: null, classified: "unavailable" };
|
|
5382
|
+
}
|
|
3868
5383
|
try {
|
|
3869
|
-
const stdout = await runGh(args, { operation });
|
|
5384
|
+
const stdout = await runGh(args, { operation, input });
|
|
3870
5385
|
return { name, args, ms: Date.now() - startedAt, bytes: stdout.length, classified: "ok" };
|
|
3871
5386
|
} catch (err) {
|
|
3872
5387
|
return {
|
|
@@ -3947,6 +5462,28 @@ function targetSource() {
|
|
|
3947
5462
|
// measures as free (verified: delta 0), so this is safe to run on a diagnostic
|
|
3948
5463
|
// path. GHES tenants can be configured with a different ceiling, which is
|
|
3949
5464
|
// exactly why this reports the server's own numbers rather than asserting 5,000.
|
|
5465
|
+
// The claimed GraphQL observer, and the only source of spendable GraphQL
|
|
5466
|
+
// capacity. `/rate_limit` used to supply this number for free, but a counter
|
|
5467
|
+
// obtained for free is not authority: it is another endpoint's view of the
|
|
5468
|
+
// meter, it can lag behind the spend it is meant to bound, and it can never
|
|
5469
|
+
// report what a particular query actually cost. This query costs a point and
|
|
5470
|
+
// says exactly what it cost, which is what makes it reconcilable.
|
|
5471
|
+
async function readGraphqlObserver(signal, host = effectiveRuntimeHost(), {
|
|
5472
|
+
fetchPage = fetchGraphqlPage,
|
|
5473
|
+
} = {}) {
|
|
5474
|
+
const page = await fetchPage("observer", { signal, operation: "graphql-observer", host, variables: {} });
|
|
5475
|
+
const observed = page.observations[0];
|
|
5476
|
+
if (!observed || observed.resource !== "graphql") return null;
|
|
5477
|
+
return {
|
|
5478
|
+
budget: { remaining: observed.remaining, limit: observed.limit, used: observed.used, resetMs: observed.resetMs },
|
|
5479
|
+
receivedAt: observed.receivedAt,
|
|
5480
|
+
cost: page.observedCost,
|
|
5481
|
+
};
|
|
5482
|
+
}
|
|
5483
|
+
|
|
5484
|
+
// Report data only, and explicitly not authority: it is left available because
|
|
5485
|
+
// a diagnostic that can contradict the observer is useful evidence, and pinned
|
|
5486
|
+
// as non-authoritative so it can never be mistaken for spendable capacity.
|
|
3950
5487
|
async function readRateLimitResources(signal, host = effectiveRuntimeHost()) {
|
|
3951
5488
|
const raw = await runGh(["api", "rate_limit", ...apiHostArgs(host)], {
|
|
3952
5489
|
signal,
|
|
@@ -4037,20 +5574,21 @@ async function readSharedBudgetSources(signal, host = effectiveRuntimeHost(), {
|
|
|
4037
5574
|
resources = RATE_RESOURCES,
|
|
4038
5575
|
coreEtag = null,
|
|
4039
5576
|
renewClaim = null,
|
|
4040
|
-
readGraphql =
|
|
5577
|
+
readGraphql = readGraphqlObserver,
|
|
4041
5578
|
readCore = readCoreBudget,
|
|
4042
5579
|
} = {}) {
|
|
4043
5580
|
const result = {};
|
|
4044
5581
|
if (resources.includes("graphql")) {
|
|
4045
|
-
const
|
|
4046
|
-
|
|
4047
|
-
|
|
4048
|
-
result.graphql = { budget: graphql, receivedAt: Date.now(), cost: 0 };
|
|
5582
|
+
const observed = await readGraphql(signal, host).catch(() => null);
|
|
5583
|
+
if (!observed?.budget) return null;
|
|
5584
|
+
result.graphql = observed;
|
|
4049
5585
|
}
|
|
4050
5586
|
if (resources.includes("core")) {
|
|
4051
|
-
// Each gh command is independently bounded, but two sequential commands
|
|
4052
|
-
//
|
|
4053
|
-
//
|
|
5587
|
+
// Each gh command is independently bounded, but two sequential commands can
|
|
5588
|
+
// outlive one claim. Extend the same nonce between the two observers. The
|
|
5589
|
+
// GraphQL read is no longer the free one -- it costs a point of its own now
|
|
5590
|
+
// -- so this renewal separates two charged requests, not a free one from a
|
|
5591
|
+
// charged one.
|
|
4054
5592
|
if (resources.includes("graphql") && typeof renewClaim === "function" && !await renewClaim()) return null;
|
|
4055
5593
|
const core = await readCore(signal, host, coreEtag);
|
|
4056
5594
|
if (core) result.core = core;
|
|
@@ -4072,12 +5610,21 @@ function projectedHourlyCost(activeKey) {
|
|
|
4072
5610
|
}
|
|
4073
5611
|
|
|
4074
5612
|
function doctorProbePlan() {
|
|
5613
|
+
// GraphQL cannot infer a repository the way `gh` does, so a report run from a
|
|
5614
|
+
// directory with no unambiguous remote says so per probe instead of sending a
|
|
5615
|
+
// document with a missing variable and reporting the resulting error as if it
|
|
5616
|
+
// came from the server.
|
|
5617
|
+
const target = effectiveRuntimeRepository();
|
|
5618
|
+
const document = (kind, variables) => (variables ? graphqlInput(kind, variables) : UNRESOLVED_REPOSITORY);
|
|
4075
5619
|
return [
|
|
4076
|
-
|
|
5620
|
+
// Every probe is now an explicit request whose cost is declared. The old
|
|
5621
|
+
// Issues/PRs entries ran `gh issue list`/`gh pr list`, whose `--search`
|
|
5622
|
+
// routed through GraphQL at a price the report could not name.
|
|
5623
|
+
["Repository access", graphqlArgs(), "doctor:repository", document("repository", graphqlRepositoryVariables(target)), "repository"],
|
|
4077
5624
|
["Actions runs", ghApiArgs(actionsRunsArgs(MIN_RUN_LIMIT)), "doctor:actions-runs"],
|
|
4078
5625
|
["Actions workflows", ghApiArgs(actionsWorkflowsArgs()), "doctor:actions-workflows"],
|
|
4079
|
-
["Issues (
|
|
4080
|
-
["Pull requests (
|
|
5626
|
+
["Issues (first page)", graphqlArgs(), "doctor:issues", document("issues", graphqlPageVariables(null, target)), "issues"],
|
|
5627
|
+
["Pull requests (first page)", graphqlArgs(), "doctor:prs", document("prs", graphqlPageVariables(null, target)), "prs"],
|
|
4081
5628
|
...ALERT_SOURCES.flatMap((source) =>
|
|
4082
5629
|
alertRequestArgs(source).map((args, index) => [
|
|
4083
5630
|
index === 0 ? source.name : `${source.name} (priority ${index})`,
|
|
@@ -4128,18 +5675,23 @@ async function runDoctor() {
|
|
|
4128
5675
|
|
|
4129
5676
|
// Free checks run first. The display rate_limit read is report data only;
|
|
4130
5677
|
// admission re-reads its GraphQL source after owning the shared claim.
|
|
4131
|
-
const [ghVersion,
|
|
5678
|
+
const [ghVersion, remote, remoteUrls] = await Promise.all([
|
|
4132
5679
|
captureGh(["--version"], "version"),
|
|
4133
|
-
captureGh(["auth", "status"], "auth-status"),
|
|
4134
5680
|
gitRemote(),
|
|
4135
5681
|
gitRemoteUrls(),
|
|
4136
5682
|
]);
|
|
4137
5683
|
const effectiveHost = effectiveRuntimeHost({ remoteUrls });
|
|
4138
|
-
|
|
5684
|
+
runtimeIdentityCoordinator = createIdentityCoordinator({ host: effectiveHost });
|
|
5685
|
+
const verified = await runtimeIdentityCoordinator.refresh();
|
|
5686
|
+
const verifiedIdentity = runtimeIdentityCoordinator.current();
|
|
5687
|
+
const authStatus = verifiedIdentity ? `${verifiedIdentity.host}: ${verifiedIdentity.login} (verified)` : identityCoordinationMessage(verified.reason);
|
|
5688
|
+
const resources = verifiedIdentity ? await readRateLimitResources(undefined, effectiveHost).catch(() => null) : null;
|
|
4139
5689
|
const budget = resources
|
|
4140
5690
|
? await rateBudget(resources)
|
|
4141
5691
|
: { core: "unavailable", graphql: "unavailable" };
|
|
4142
|
-
const scopeResult =
|
|
5692
|
+
const scopeResult = verifiedIdentity
|
|
5693
|
+
? { ok: true, value: createQuotaScope(verifiedIdentity, { root: runtimeIdentityCoordinator.root, identityProvider: runtimeIdentityCoordinator.current }) }
|
|
5694
|
+
: verified;
|
|
4143
5695
|
let governorResult = scopeResult;
|
|
4144
5696
|
let results = [];
|
|
4145
5697
|
if (scopeResult.ok) {
|
|
@@ -4157,20 +5709,21 @@ async function runDoctor() {
|
|
|
4157
5709
|
try {
|
|
4158
5710
|
if (registered.ok) await refreshSharedBudget(scope, leaseId, undefined);
|
|
4159
5711
|
const admissionAt = Date.now();
|
|
4160
|
-
const admittedProbes = probes.map(([name, args, operation], index) => {
|
|
5712
|
+
const admittedProbes = probes.map(([name, args, operation, input = null, document = null], index) => {
|
|
4161
5713
|
const admitted = registered.ok
|
|
4162
5714
|
? admitGovernorOperation(scope, leaseId, operation, "diagnostic", admissionAt)
|
|
4163
5715
|
: registered;
|
|
4164
5716
|
if (!admitted?.ok || admitted.value.status !== "started") {
|
|
4165
|
-
return { index, skipped: skippedDoctorProbe(name, args, admitted) };
|
|
5717
|
+
return { index, skipped: { ...skippedDoctorProbe(name, args, admitted), ...(document ? { document } : {}) } };
|
|
4166
5718
|
}
|
|
4167
|
-
return { index, name, args, operation, reservationId: admitted.value.reservationId };
|
|
5719
|
+
return { index, name, args, operation, input, document, reservationId: admitted.value.reservationId };
|
|
4168
5720
|
});
|
|
4169
5721
|
results = admittedProbes.map((item) => item.skipped ?? null);
|
|
4170
5722
|
const runnable = admittedProbes.filter((item) => !item.skipped);
|
|
4171
5723
|
const settled = await mapAllSettledBounded(runnable, 4, async (item) => {
|
|
4172
|
-
const result = await probe(item.name, item.args, item.operation);
|
|
4173
|
-
|
|
5724
|
+
const result = await requestIdentityStorage.run(scope, () => probe(item.name, item.args, item.operation, item.input));
|
|
5725
|
+
if (item.document) result.document = item.document;
|
|
5726
|
+
completeReservation({ ...scope, identityProvider: null }, item.reservationId, result.failed
|
|
4174
5727
|
? { outcome: "rejected" }
|
|
4175
5728
|
: { outcome: "measured-success", actualCost: operationCost(item.operation) }, Date.now());
|
|
4176
5729
|
return result;
|
|
@@ -4211,7 +5764,12 @@ async function runDoctor() {
|
|
|
4211
5764
|
field("slug", runtime.repo ?? "(inferred from the working directory)"),
|
|
4212
5765
|
field("git remote", remote),
|
|
4213
5766
|
"",
|
|
4214
|
-
|
|
5767
|
+
// Labelled, because these two numbers come from `gh api rate_limit`, which
|
|
5768
|
+
// this version treats as a diagnostic and never as spendable capacity. The
|
|
5769
|
+
// "API governor" section below carries the numbers actually admitted
|
|
5770
|
+
// against, with their provenance. A reader who cannot tell the two apart
|
|
5771
|
+
// will believe the wrong one.
|
|
5772
|
+
...section("API budget (rate_limit, non-authoritative)"),
|
|
4215
5773
|
field("REST core", budget.core),
|
|
4216
5774
|
field("GraphQL", budget.graphql),
|
|
4217
5775
|
field(
|
|
@@ -4246,6 +5804,10 @@ async function runDoctor() {
|
|
|
4246
5804
|
for (const result of results) {
|
|
4247
5805
|
lines.push(` ${result.name}`);
|
|
4248
5806
|
probeLine("argv", `gh ${result.args.join(" ")}`);
|
|
5807
|
+
// Three GraphQL probes share one argv, and the document that distinguishes
|
|
5808
|
+
// them travels on stdin. A report that cannot say which query it sent is
|
|
5809
|
+
// exactly the drift this command exists to rule out.
|
|
5810
|
+
if (result.document) probeLine("document", result.document);
|
|
4249
5811
|
probeLine(
|
|
4250
5812
|
"outcome",
|
|
4251
5813
|
result.skipped
|
|
@@ -4626,6 +6188,21 @@ if (IS_MAIN) {
|
|
|
4626
6188
|
console.error(problem);
|
|
4627
6189
|
process.exit(3);
|
|
4628
6190
|
}
|
|
6191
|
+
runtimeRemoteUrls = await gitRemoteUrls();
|
|
6192
|
+
runtimeIdentityCoordinator = createIdentityCoordinator({ host: () => effectiveRuntimeHost() });
|
|
6193
|
+
// Resolve warm identity/cache locally; cold proof belongs to the mounted UI
|
|
6194
|
+
// so quit/signal handling remains available while GitHub is slow.
|
|
6195
|
+
//
|
|
6196
|
+
// Bounded, because the warm path still spawns `gh auth token`: a prompting
|
|
6197
|
+
// credential helper, a keyring daemon that is not up yet in a fresh login
|
|
6198
|
+
// session, or a cold binary on a network filesystem would otherwise hold the
|
|
6199
|
+
// first frame for the full subprocess timeout with nothing on screen at all.
|
|
6200
|
+
// Past the bound the same refresh is picked up by the mounted UI, and
|
|
6201
|
+
// ensureScope corrects the cache target and hydrates from it.
|
|
6202
|
+
await Promise.race([
|
|
6203
|
+
runtimeIdentityCoordinator.refresh({ allowBootstrap: false }),
|
|
6204
|
+
new Promise((resolve) => { setTimeout(resolve, WARM_IDENTITY_WAIT_MS).unref(); }),
|
|
6205
|
+
]);
|
|
4629
6206
|
}
|
|
4630
6207
|
|
|
4631
6208
|
const ReactModule = await import("react");
|
|
@@ -5309,7 +6886,7 @@ const TABS = [
|
|
|
5309
6886
|
},
|
|
5310
6887
|
{
|
|
5311
6888
|
key: "issues",
|
|
5312
|
-
fetch: ({ signal }) => fetchIssues(signal),
|
|
6889
|
+
fetch: ({ signal, governor, previousRaw }) => fetchIssues(signal, { governor, previousRaw }),
|
|
5313
6890
|
label: "Issues",
|
|
5314
6891
|
short: "Issues",
|
|
5315
6892
|
header: ISSUES_HEADER,
|
|
@@ -5319,7 +6896,7 @@ const TABS = [
|
|
|
5319
6896
|
},
|
|
5320
6897
|
{
|
|
5321
6898
|
key: "prs",
|
|
5322
|
-
fetch: ({ signal }) => fetchPRs(signal),
|
|
6899
|
+
fetch: ({ signal, governor, previousRaw }) => fetchPRs(signal, { governor, previousRaw }),
|
|
5323
6900
|
label: "Pull requests",
|
|
5324
6901
|
short: "PRs",
|
|
5325
6902
|
header: PRS_HEADER,
|
|
@@ -5869,7 +7446,8 @@ const createWidthPreferenceWriter = createCoalescedWriter;
|
|
|
5869
7446
|
|
|
5870
7447
|
// ---------- Last-known-good dashboard cache ----------
|
|
5871
7448
|
|
|
5872
|
-
|
|
7449
|
+
// 3: cached rows carry their page URL, without which they cannot be opened.
|
|
7450
|
+
const DASHBOARD_CACHE_VERSION = 3;
|
|
5873
7451
|
const MAX_DASHBOARD_CACHE_TARGETS = 5;
|
|
5874
7452
|
const MAX_DASHBOARD_CACHE_ROWS_PER_TAB = 60;
|
|
5875
7453
|
let dashboardCacheTempSequence = 0;
|
|
@@ -5958,6 +7536,9 @@ function normalizeCachedItem(tabKey, item) {
|
|
|
5958
7536
|
conclusion: item.conclusion == null ? null : safe(item.conclusion),
|
|
5959
7537
|
startedAt: safe(item.startedAt),
|
|
5960
7538
|
updatedAt: safe(item.updatedAt),
|
|
7539
|
+
// Retained because opening a row now reads its URL from the row. Dropping
|
|
7540
|
+
// it made Enter on a cache-hydrated row do nothing at all, with no error.
|
|
7541
|
+
url: safe(item.url ?? ""),
|
|
5961
7542
|
};
|
|
5962
7543
|
}
|
|
5963
7544
|
if (tabKey === "issues") {
|
|
@@ -5968,6 +7549,7 @@ function normalizeCachedItem(tabKey, item) {
|
|
|
5968
7549
|
author: safe(item.author),
|
|
5969
7550
|
label: safe(item.label),
|
|
5970
7551
|
updatedAt: safe(item.updatedAt),
|
|
7552
|
+
url: safe(item.url ?? ""),
|
|
5971
7553
|
};
|
|
5972
7554
|
}
|
|
5973
7555
|
if (tabKey === "prs") {
|
|
@@ -5980,6 +7562,7 @@ function normalizeCachedItem(tabKey, item) {
|
|
|
5980
7562
|
isDraft: Boolean(item.isDraft),
|
|
5981
7563
|
reviewDecision: safe(item.reviewDecision),
|
|
5982
7564
|
updatedAt: safe(item.updatedAt),
|
|
7565
|
+
url: safe(item.url ?? ""),
|
|
5983
7566
|
};
|
|
5984
7567
|
}
|
|
5985
7568
|
if (tabKey === "security") {
|
|
@@ -6260,14 +7843,20 @@ function retryPollAfterAdmissionFailure({
|
|
|
6260
7843
|
|
|
6261
7844
|
function governorWakeTimes(state, nowMs, floorMs, leaseId = null) {
|
|
6262
7845
|
const budgets = Object.values(state?.budgets ?? {});
|
|
6263
|
-
|
|
6264
|
-
|
|
6265
|
-
|
|
7846
|
+
// A failed observer's own retry, not its stale sample's cadence, says when
|
|
7847
|
+
// that resource is next worth reading. Suppress the cadence per resource: a
|
|
7848
|
+
// global suppression let one failure silence a healthy lane's wake, and no
|
|
7849
|
+
// suppression let the failed lane spin on the sample it already knows is old.
|
|
7850
|
+
const observedCandidates = RATE_RESOURCES.flatMap((resource) => {
|
|
7851
|
+
const budget = state?.budgets?.[resource];
|
|
7852
|
+
if (!budget || state?.observers?.[resource]?.outcome === "failed") return [];
|
|
7853
|
+
return [budget.observedAt + BUDGET_PROBE_MS];
|
|
7854
|
+
});
|
|
6266
7855
|
const controlAt = Math.min(
|
|
6267
7856
|
...observedCandidates,
|
|
6268
7857
|
...budgets.map((budget) => budget.resetMs + BUDGET_RESET_GRACE_MS),
|
|
6269
|
-
state?.
|
|
6270
|
-
state?.
|
|
7858
|
+
...RATE_RESOURCES.map((resource) => state?.probeClaims?.[resource]?.leaseUntil ?? Number.POSITIVE_INFINITY),
|
|
7859
|
+
...RATE_RESOURCES.map((resource) => state?.observers?.[resource]?.nextAt ?? Number.POSITIVE_INFINITY),
|
|
6271
7860
|
);
|
|
6272
7861
|
const reservationAt = Object.values(state?.reservations ?? {})
|
|
6273
7862
|
.filter((reservation) => reservation.status === "scheduled" &&
|
|
@@ -6282,7 +7871,8 @@ function governorWakeTimes(state, nowMs, floorMs, leaseId = null) {
|
|
|
6282
7871
|
function governorProtocolReady(refreshResult, snapshot, nowMs) {
|
|
6283
7872
|
if (!refreshResult?.ok || !snapshot?.ok) return false;
|
|
6284
7873
|
if (["waiting", "paused", "probe"].includes(refreshResult.value?.status)) return false;
|
|
6285
|
-
if (
|
|
7874
|
+
if (RATE_RESOURCES.some((resource) => snapshot.value.probeClaims[resource]) ||
|
|
7875
|
+
RATE_RESOURCES.some((resource) => snapshot.value.observers[resource]?.outcome !== "healthy")) return false;
|
|
6286
7876
|
return RATE_RESOURCES.every((resource) => {
|
|
6287
7877
|
const budget = snapshot.value.budgets[resource];
|
|
6288
7878
|
return budget && nowMs - budget.observedAt <= budgetSnapshotTtl(resource);
|
|
@@ -6439,10 +8029,27 @@ function rateLimitBlockDecision(results, resetMs) {
|
|
|
6439
8029
|
};
|
|
6440
8030
|
}
|
|
6441
8031
|
|
|
8032
|
+
// Identity reasons that name a condition the user can act on or wait out. The
|
|
8033
|
+
// list previously stopped short of identity-unavailable -- the ordinary "not
|
|
8034
|
+
// signed in" or "proof denied" case, and precisely the one this phase requires
|
|
8035
|
+
// to read as an honest unavailable state rather than a generic retry -- and of
|
|
8036
|
+
// identity-capacity and transport-busy.
|
|
8037
|
+
//
|
|
8038
|
+
// Purely internal reasons (corrupt, unwritable, unknown-host, closed) stay on
|
|
8039
|
+
// the generic message on purpose: this surface must not leak internal
|
|
8040
|
+
// vocabulary, which is the contract "coordination notices translate raw reasons
|
|
8041
|
+
// without exposing internal vocabulary" in test/unit.test.mjs pins.
|
|
8042
|
+
const IDENTITY_COORDINATION_REASONS = [
|
|
8043
|
+
"restart-required", "migration-hold", "legacy-unresolved", "legacy-corrupt",
|
|
8044
|
+
"credential-unavailable", "identity-backoff", "identity-busy",
|
|
8045
|
+
"identity-unavailable", "identity-capacity", "transport-busy", "throttle-paused",
|
|
8046
|
+
];
|
|
8047
|
+
|
|
6442
8048
|
function coordinationNotice(reason) {
|
|
6443
8049
|
if (reason === "unknown-scope") return "Confirming your GitHub login…";
|
|
6444
8050
|
if (reason === "block-unpublished") return "Holding until the rate-limit block is shared";
|
|
6445
8051
|
if (reason === "busy" || reason === "stale") return "Coordinating with your other panes";
|
|
8052
|
+
if (IDENTITY_COORDINATION_REASONS.includes(reason)) return identityCoordinationMessage(reason);
|
|
6446
8053
|
return "Can't coordinate API use — retrying";
|
|
6447
8054
|
}
|
|
6448
8055
|
|
|
@@ -6523,16 +8130,18 @@ async function runAdmittedOperation({ scope, leaseId, operation, priority = "man
|
|
|
6523
8130
|
return { ok: false, skipped: true, decision: admitted, error: new Error(`API budget paused${detail}`) };
|
|
6524
8131
|
}
|
|
6525
8132
|
const reservationId = admitted.value.reservationId;
|
|
8133
|
+
const settlementScope = { ...scope, identityProvider: null };
|
|
6526
8134
|
try {
|
|
6527
|
-
const value = await run(signal);
|
|
8135
|
+
const value = await requestIdentityStorage.run(scope, () => run(signal));
|
|
6528
8136
|
const costs = operationCost(operation);
|
|
6529
|
-
completeReservation(
|
|
8137
|
+
completeReservation(settlementScope, reservationId, {
|
|
6530
8138
|
outcome: "measured-success",
|
|
6531
8139
|
actualCost: costs,
|
|
6532
8140
|
}, Date.now());
|
|
8141
|
+
if (scope.accessKey && scope.accessKey !== runtimeIdentityCoordinator?.current()?.accessKey) return { ok: false, error: new Error("Credential changed"), reservationId };
|
|
6533
8142
|
return { ok: true, value, reservationId };
|
|
6534
8143
|
} catch (error) {
|
|
6535
|
-
completeReservation(
|
|
8144
|
+
completeReservation(settlementScope, reservationId, { outcome: governorOutcomeForError(error) }, Date.now());
|
|
6536
8145
|
return { ok: false, error, reservationId };
|
|
6537
8146
|
}
|
|
6538
8147
|
}
|
|
@@ -7183,8 +8792,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7183
8792
|
dashboardCacheTarget({
|
|
7184
8793
|
repo: runtime.repo,
|
|
7185
8794
|
ghRepo: process.env.GH_REPO,
|
|
7186
|
-
host:
|
|
8795
|
+
host: runtimeIdentityCoordinator?.current()?.host ?? effectiveRuntimeHost(),
|
|
7187
8796
|
cwd: process.cwd(),
|
|
8797
|
+
account: runtimeIdentityCoordinator?.current()?.accessKey ?? "unverified",
|
|
7188
8798
|
}),
|
|
7189
8799
|
);
|
|
7190
8800
|
const [loadedCache] = useState(() => loadDashboardCache(cachePath, cacheTarget));
|
|
@@ -7885,6 +9495,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7885
9495
|
// refresh the tab you just landed on without waiting on an unrelated
|
|
7886
9496
|
// background fetch -- and so a slow repo can't stack refreshes.
|
|
7887
9497
|
if (inFlightRef.current[key]) return Promise.resolve();
|
|
9498
|
+
const settlement = createSettlementContext(scope, runtimeIdentityCoordinator);
|
|
9499
|
+
const settlementScope = settlement.scope;
|
|
9500
|
+
const currentAccess = settlement.isCurrent;
|
|
7888
9501
|
inFlightRef.current[key] = true;
|
|
7889
9502
|
if (force) manualInFlight.add(key);
|
|
7890
9503
|
admittedCadenceRef.current[key] = nextAdmittedCadence(
|
|
@@ -7910,9 +9523,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7910
9523
|
[key]: { automaticStatusVisible },
|
|
7911
9524
|
}));
|
|
7912
9525
|
}
|
|
7913
|
-
return run()
|
|
9526
|
+
return requestIdentityStorage.run(scope, run)
|
|
7914
9527
|
.then((result) => {
|
|
7915
|
-
settleReservationWithBudgetObservations(
|
|
9528
|
+
settleReservationWithBudgetObservations(settlementScope, leaseId, reservationId, result?.measuredSuccess === false
|
|
7916
9529
|
? { outcome: "rejected", observations: result?.observations ?? [] }
|
|
7917
9530
|
: {
|
|
7918
9531
|
outcome: "measured-success",
|
|
@@ -7922,6 +9535,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
7922
9535
|
},
|
|
7923
9536
|
observations: result?.observations ?? [],
|
|
7924
9537
|
}, Date.now());
|
|
9538
|
+
if (!currentAccess()) return;
|
|
7925
9539
|
if (result?.rateLimited) {
|
|
7926
9540
|
const blockAt = Date.now();
|
|
7927
9541
|
const budget = inspectGovernor(scope, blockAt).value?.budgets?.core;
|
|
@@ -8038,11 +9652,11 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8038
9652
|
});
|
|
8039
9653
|
})
|
|
8040
9654
|
.catch((err) => {
|
|
8041
|
-
settleReservationWithBudgetObservations(
|
|
9655
|
+
settleReservationWithBudgetObservations(settlementScope, leaseId, reservationId, {
|
|
8042
9656
|
outcome: governorOutcomeForError(err),
|
|
8043
9657
|
observations: err?.budgetObservations ?? [],
|
|
8044
9658
|
}, Date.now());
|
|
8045
|
-
if (cancelled || err?.name === "AbortError") return;
|
|
9659
|
+
if (cancelled || !currentAccess() || err?.name === "AbortError") return;
|
|
8046
9660
|
// Preserve both the verdict and the bounded raw error in state. The
|
|
8047
9661
|
// renderer translates recognized verdicts at draw time, which lets a
|
|
8048
9662
|
// later repository/account context refine the one-line remedy without
|
|
@@ -8076,6 +9690,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8076
9690
|
})
|
|
8077
9691
|
.finally(() => {
|
|
8078
9692
|
inFlightRef.current[key] = false;
|
|
9693
|
+
if (!currentAccess() && !Object.values(inFlightRef.current).some(Boolean) && runtimeIdentityCoordinator?.current()?.quotaKey !== settlementScope.quotaKey) releaseLease(settlementScope, leaseId);
|
|
8079
9694
|
manualInFlight.delete(key);
|
|
8080
9695
|
onSettled?.();
|
|
8081
9696
|
if (!cancelled) {
|
|
@@ -8109,6 +9724,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8109
9724
|
let scope = null;
|
|
8110
9725
|
let cleanupScope = null;
|
|
8111
9726
|
let registeredScopeHash = null;
|
|
9727
|
+
// The access partition the retained rows belong to, kept across a window
|
|
9728
|
+
// where the identity is momentarily unknown so that an account which comes
|
|
9729
|
+
// back *different* still clears them. Null means nothing is retained.
|
|
9730
|
+
let retainedAccessKey = null;
|
|
8112
9731
|
let remoteUrls = [];
|
|
8113
9732
|
let liveScheduling = false;
|
|
8114
9733
|
let controlEpochs = null;
|
|
@@ -8178,14 +9797,34 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8178
9797
|
}
|
|
8179
9798
|
|
|
8180
9799
|
function identity() {
|
|
8181
|
-
return
|
|
8182
|
-
|
|
8183
|
-
|
|
8184
|
-
|
|
9800
|
+
return runtimeIdentityCoordinator?.current() ?? null;
|
|
9801
|
+
}
|
|
9802
|
+
|
|
9803
|
+
function retireCurrentScope() {
|
|
9804
|
+
if (!cleanupScope || !registeredScopeHash) return;
|
|
9805
|
+
for (const item of pending.values()) cancelIntent(cleanupScope, item.intentId, Date.now());
|
|
9806
|
+
if (!Object.values(inFlightRef.current).some(Boolean)) releaseLease(cleanupScope, leaseId);
|
|
8185
9807
|
}
|
|
8186
9808
|
|
|
8187
|
-
|
|
9809
|
+
// discardData separates the two reasons this runs. An account *change* must
|
|
9810
|
+
// drop every row, validator and note, because they belong to an access
|
|
9811
|
+
// partition this pane no longer holds. An identity that is merely unknown
|
|
9812
|
+
// for a moment must not: the rows are still ours, and throwing away the
|
|
9813
|
+
// ETags with them makes the recovery round cost full price.
|
|
9814
|
+
function resetVisibleScope(nowMs, { immediate = false, discardData = true } = {}) {
|
|
8188
9815
|
pending.clear();
|
|
9816
|
+
coordinator.invalidate();
|
|
9817
|
+
setFailureContext(null);
|
|
9818
|
+
if (discardData) {
|
|
9819
|
+
rawRef.current = {};
|
|
9820
|
+
entityRef.current.clear();
|
|
9821
|
+
lastOkRef.current = {};
|
|
9822
|
+
setData(Object.fromEntries(TAB_KEYS.map((key) => [key, null])));
|
|
9823
|
+
setMeta({});
|
|
9824
|
+
setSecurityNotes([]);
|
|
9825
|
+
setSecurityBlind(false);
|
|
9826
|
+
retainedAccessKey = null;
|
|
9827
|
+
}
|
|
8189
9828
|
pendingBlockPublications.clear();
|
|
8190
9829
|
liveScheduling = false;
|
|
8191
9830
|
activePollAt = Number.POSITIVE_INFINITY;
|
|
@@ -8204,20 +9843,26 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8204
9843
|
|
|
8205
9844
|
function ensureScope(nowMs = Date.now(), { maintain = false } = {}) {
|
|
8206
9845
|
const current = identity();
|
|
8207
|
-
const nextHash =
|
|
9846
|
+
const nextHash = current?.quotaKey;
|
|
8208
9847
|
if (!nextHash) {
|
|
8209
9848
|
if (scope !== null) {
|
|
8210
|
-
|
|
9849
|
+
retireCurrentScope();
|
|
8211
9850
|
registeredScopeHash = null;
|
|
8212
9851
|
if (governorRef.current?.leaseId === leaseId) governorRef.current = null;
|
|
8213
9852
|
scope = null;
|
|
8214
9853
|
cleanupScope = null;
|
|
8215
|
-
|
|
9854
|
+
// Unknown is not the same as changed. A contended registry lock, a
|
|
9855
|
+
// `gh` write that only touched an unrelated host's entry, or a
|
|
9856
|
+
// re-resolution still in flight all arrive here, and none of them says
|
|
9857
|
+
// the rows on screen stopped being ours. Stop scheduling and let them
|
|
9858
|
+
// age visibly; retainedAccessKey is what still clears them if the
|
|
9859
|
+
// account turns out to have actually changed.
|
|
9860
|
+
resetVisibleScope(nowMs, { discardData: false });
|
|
8216
9861
|
}
|
|
8217
9862
|
return null;
|
|
8218
9863
|
}
|
|
8219
|
-
const migrating = scope !== null && scope.hash !== nextHash;
|
|
8220
|
-
if (scope?.hash === nextHash && registeredScopeHash === nextHash) {
|
|
9864
|
+
const migrating = scope !== null && (scope.hash !== nextHash || scope.accessKey !== current.accessKey);
|
|
9865
|
+
if (scope?.hash === nextHash && scope.accessKey === current.accessKey && registeredScopeHash === nextHash) {
|
|
8221
9866
|
if (!maintain) return scope;
|
|
8222
9867
|
const activeTab = TABS[activeIndexRef.current].key;
|
|
8223
9868
|
const kept = maintainControlLease(scope, leaseId, runtime.refreshMs, activeTab, nowMs);
|
|
@@ -8226,20 +9871,40 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8226
9871
|
if (governorRef.current?.leaseId === leaseId) governorRef.current = null;
|
|
8227
9872
|
return null;
|
|
8228
9873
|
}
|
|
8229
|
-
if (scope?.hash !== nextHash) {
|
|
8230
|
-
|
|
9874
|
+
if (scope?.hash !== nextHash || scope.accessKey !== current.accessKey) {
|
|
9875
|
+
retireCurrentScope();
|
|
8231
9876
|
registeredScopeHash = null;
|
|
8232
|
-
|
|
9877
|
+
// The second test covers the identity returning after a window where it
|
|
9878
|
+
// was unknown: scope is null by then, so `migrating` cannot see the
|
|
9879
|
+
// change, and retained rows from the previous access partition would
|
|
9880
|
+
// otherwise survive into an account that must not be able to read them.
|
|
9881
|
+
if (migrating || (scope === null && retainedAccessKey !== null && retainedAccessKey !== current.accessKey)) {
|
|
8233
9882
|
// The failed publication belonged to the old account scope. Releasing
|
|
8234
9883
|
// that lease guarantees this process will start no more work there;
|
|
8235
9884
|
// the new scope must establish its own authoritative budget instead
|
|
8236
9885
|
// of inheriting an unrelated account's local hold.
|
|
8237
9886
|
resetVisibleScope(nowMs, { immediate: true });
|
|
8238
9887
|
}
|
|
8239
|
-
|
|
8240
|
-
if (!created.ok) return null;
|
|
8241
|
-
scope = created.value;
|
|
9888
|
+
scope = createQuotaScope(current, { root: runtimeIdentityCoordinator.root, identityProvider: identity });
|
|
8242
9889
|
cleanupScope = { ...scope, identityProvider: null };
|
|
9890
|
+
const nextTarget = dashboardCacheTarget({ repo: runtime.repo, ghRepo: process.env.GH_REPO, host: current.host, account: current.accessKey });
|
|
9891
|
+
// The mount-time target had to guess the account, and guesses
|
|
9892
|
+
// "unverified" whenever the bounded warm resolution above did not land
|
|
9893
|
+
// in time -- so a session that has rows on disk can start with none.
|
|
9894
|
+
// Now that the partition is proven, take them. Guarded on having
|
|
9895
|
+
// published nothing yet, so this can never overwrite live rows.
|
|
9896
|
+
if (nextTarget !== dashboardCacheTargetRef.current && retainedAccessKey === null &&
|
|
9897
|
+
Object.values(dataRef.current).every((rows) => rows === null)) {
|
|
9898
|
+
const warm = pick(dashboardCacheRef.current, nextTarget, null);
|
|
9899
|
+
if (warm) {
|
|
9900
|
+
setData(Object.fromEntries(TABS.map((candidate) => [candidate.key, warm.tabs[candidate.key]?.data ?? null])));
|
|
9901
|
+
setMeta(Object.fromEntries(TABS.map((candidate) => [candidate.key, warm.tabs[candidate.key]?.meta ?? null])));
|
|
9902
|
+
setSecurityNotes(warm.securityNotes ?? []);
|
|
9903
|
+
setSecurityBlind(warm.securityBlind ?? false);
|
|
9904
|
+
}
|
|
9905
|
+
}
|
|
9906
|
+
retainedAccessKey = current.accessKey;
|
|
9907
|
+
dashboardCacheTargetRef.current = nextTarget;
|
|
8243
9908
|
}
|
|
8244
9909
|
const activeTab = TABS[activeIndexRef.current].key;
|
|
8245
9910
|
const registered = registerLease(scope, {
|
|
@@ -8416,6 +10081,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8416
10081
|
runLimit: runLimitRef.current,
|
|
8417
10082
|
entities: entityRef.current,
|
|
8418
10083
|
force,
|
|
10084
|
+
governor: { scope: currentScope, leaseId },
|
|
8419
10085
|
previousRaw: rawRef.current[key] ?? null,
|
|
8420
10086
|
}),
|
|
8421
10087
|
{
|
|
@@ -8455,13 +10121,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8455
10121
|
if (inFlightRef.current[key]) continue;
|
|
8456
10122
|
const currentScope = ensureScope(nowMs);
|
|
8457
10123
|
if (!currentScope) continue;
|
|
8458
|
-
const decision = readIntentDecision(currentScope, item.intentId, nowMs);
|
|
10124
|
+
const decision = readIntentDecision(currentScope, item.intentId, nowMs, item);
|
|
8459
10125
|
if (!decision.ok) {
|
|
8460
10126
|
pauseCoordination(key, decision.reason);
|
|
8461
10127
|
if (pendingFailureIsTerminal(decision.reason)) pending.delete(key);
|
|
8462
10128
|
else armWake("data", governorControlRetryAt(nowMs, runtime.refreshMs), dataWake);
|
|
8463
10129
|
continue;
|
|
8464
10130
|
}
|
|
10131
|
+
Object.assign(item, sharedLaneEvidence(decision.value));
|
|
8465
10132
|
if (decision.value.status !== "scheduled" || decision.value.notBefore > nowMs) {
|
|
8466
10133
|
item.wasDeferred = true;
|
|
8467
10134
|
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
@@ -8501,6 +10168,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8501
10168
|
runLimit: runLimitRef.current,
|
|
8502
10169
|
entities: entityRef.current,
|
|
8503
10170
|
force: item.force,
|
|
10171
|
+
governor: { scope: currentScope, leaseId },
|
|
8504
10172
|
previousRaw: rawRef.current[key] ?? null,
|
|
8505
10173
|
}),
|
|
8506
10174
|
{
|
|
@@ -8585,6 +10253,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8585
10253
|
});
|
|
8586
10254
|
|
|
8587
10255
|
async function controlWake() {
|
|
10256
|
+
if (cancelled) return;
|
|
10257
|
+
await runtimeIdentityCoordinator.refresh();
|
|
8588
10258
|
if (cancelled) return;
|
|
8589
10259
|
const nowMs = Date.now();
|
|
8590
10260
|
const currentScope = ensureScope(nowMs, { maintain: true });
|
|
@@ -8599,7 +10269,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8599
10269
|
}
|
|
8600
10270
|
const refreshed = currentScope
|
|
8601
10271
|
? await refreshSharedBudget(currentScope, leaseId, controller.signal)
|
|
8602
|
-
: { ok: false, reason: "stale" };
|
|
10272
|
+
: { ok: false, reason: runtimeIdentityCoordinator?.inspect()?.reason ?? "stale" };
|
|
8603
10273
|
if (cancelled) return;
|
|
8604
10274
|
const checkedAt = Date.now();
|
|
8605
10275
|
const snapshot = currentScope ? inspectGovernor(currentScope, checkedAt) : refreshed;
|
|
@@ -8645,12 +10315,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
8645
10315
|
}
|
|
8646
10316
|
|
|
8647
10317
|
async function bootstrap() {
|
|
8648
|
-
remoteUrls = await gitRemoteUrls();
|
|
10318
|
+
remoteUrls = runtimeRemoteUrls.length > 0 ? runtimeRemoteUrls : await gitRemoteUrls();
|
|
8649
10319
|
runtimeRemoteUrls = remoteUrls;
|
|
10320
|
+
await runtimeIdentityCoordinator.refresh();
|
|
8650
10321
|
if (cancelled) return;
|
|
8651
10322
|
const currentScope = ensureScope(Date.now());
|
|
8652
10323
|
if (!currentScope) {
|
|
8653
|
-
pauseCoordination(TABS[activeIndexRef.current].key, "unknown-scope");
|
|
10324
|
+
pauseCoordination(TABS[activeIndexRef.current].key, runtimeIdentityCoordinator?.inspect()?.reason ?? "unknown-scope");
|
|
8654
10325
|
armWake("control", governorControlRetryAt(Date.now(), runtime.refreshMs), controlWake);
|
|
8655
10326
|
return;
|
|
8656
10327
|
}
|
|
@@ -9149,7 +10820,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
9149
10820
|
canMove: allItems.length > 0,
|
|
9150
10821
|
canOpen:
|
|
9151
10822
|
selectedKey !== null &&
|
|
9152
|
-
|
|
10823
|
+
OPENABLE.includes(tab.key) &&
|
|
9153
10824
|
allItems.some((item) => itemKey(item) === selectedKey),
|
|
9154
10825
|
canResize: fullHeaderVisible,
|
|
9155
10826
|
}),
|
|
@@ -9342,6 +11013,10 @@ if (IS_MAIN) {
|
|
|
9342
11013
|
});
|
|
9343
11014
|
};
|
|
9344
11015
|
app = render(e(App, { onCreateRemote: createRemote }), { incrementalRendering: true });
|
|
11016
|
+
// Ink's q/Esc/Ctrl+C exit unmounts directly rather than calling unmountApp.
|
|
11017
|
+
// Only close the identity coordinator after that actual application exit,
|
|
11018
|
+
// not when a polling effect reruns. Rejections reach the crash handler above.
|
|
11019
|
+
void app.waitUntilExit().then(abortLiveRequests);
|
|
9345
11020
|
|
|
9346
11021
|
// 128 + signal number, so a supervisor or `timeout` can tell an interrupted
|
|
9347
11022
|
// run from a clean one. These fire on external `kill` and when raw mode is
|
|
@@ -9371,6 +11046,7 @@ if (IS_MAIN) {
|
|
|
9371
11046
|
const finish = () => {
|
|
9372
11047
|
unmountApp();
|
|
9373
11048
|
restoreScreen();
|
|
11049
|
+
abortLiveRequests();
|
|
9374
11050
|
process.exit(code);
|
|
9375
11051
|
};
|
|
9376
11052
|
|
|
@@ -9396,6 +11072,19 @@ if (IS_MAIN) {
|
|
|
9396
11072
|
// Exported for unit tests. The dashboard itself is still one file; these are
|
|
9397
11073
|
// the pure functions worth pinning, and nothing here is part of the public API.
|
|
9398
11074
|
export {
|
|
11075
|
+
retryIdentityCompletion,
|
|
11076
|
+
createSettlementContext,
|
|
11077
|
+
startIdentityControl,
|
|
11078
|
+
settleIdentityControl,
|
|
11079
|
+
resolveEffectiveCredential,
|
|
11080
|
+
identityRegistryRoot,
|
|
11081
|
+
inspectIdentityRegistry,
|
|
11082
|
+
claimIdentityBootstrap,
|
|
11083
|
+
finishIdentityBootstrap,
|
|
11084
|
+
createIdentityCoordinator,
|
|
11085
|
+
createQuotaScope,
|
|
11086
|
+
acquireIdentityHttpPermit,
|
|
11087
|
+
releaseIdentityHttpPermit,
|
|
9399
11088
|
parseArgs,
|
|
9400
11089
|
validateArgs,
|
|
9401
11090
|
parseRepoTarget,
|
|
@@ -9438,6 +11127,7 @@ export {
|
|
|
9438
11127
|
governorPhaseOffset,
|
|
9439
11128
|
scheduleIntents,
|
|
9440
11129
|
GOVERNOR_STATE_VERSION,
|
|
11130
|
+
RATE_RESOURCES,
|
|
9441
11131
|
GOVERNOR_MAX_LEASES,
|
|
9442
11132
|
GOVERNOR_MAX_INTENTS,
|
|
9443
11133
|
GOVERNOR_MAX_RESERVATIONS,
|
|
@@ -9463,6 +11153,16 @@ export {
|
|
|
9463
11153
|
heartbeatLease,
|
|
9464
11154
|
maintainControlLease,
|
|
9465
11155
|
claimProbe,
|
|
11156
|
+
classifyThrottle,
|
|
11157
|
+
applyTransportCooldown,
|
|
11158
|
+
clearTransportThrottle,
|
|
11159
|
+
emptyTransportThrottle,
|
|
11160
|
+
throttleLadderMs,
|
|
11161
|
+
transportCooldownDeadline,
|
|
11162
|
+
retryThrottledTransport,
|
|
11163
|
+
THROTTLE_LADDER_MS,
|
|
11164
|
+
THROTTLE_PAUSE_AFTER,
|
|
11165
|
+
MANUAL_GRANT_STREAK_LIMIT,
|
|
9466
11166
|
renewProbeClaim,
|
|
9467
11167
|
publishProbe,
|
|
9468
11168
|
failProbeClaim,
|
|
@@ -9475,6 +11175,7 @@ export {
|
|
|
9475
11175
|
settleReservationWithBudgetObservations,
|
|
9476
11176
|
recordResourceBlock,
|
|
9477
11177
|
releaseLease,
|
|
11178
|
+
inspectAdmittedHttpStart,
|
|
9478
11179
|
inspectGovernor,
|
|
9479
11180
|
governorHealth,
|
|
9480
11181
|
refreshSharedBudget,
|
|
@@ -9576,6 +11277,24 @@ export {
|
|
|
9576
11277
|
forcedBackoffKeys,
|
|
9577
11278
|
clearForcedBackoffAfterStart,
|
|
9578
11279
|
doctorProbePlan,
|
|
11280
|
+
rowBrowserUrl,
|
|
11281
|
+
admittedRowUrl,
|
|
11282
|
+
remoteSlug,
|
|
11283
|
+
resolveEffectiveRepository,
|
|
11284
|
+
graphqlPageVariables,
|
|
11285
|
+
graphqlRepositoryVariables,
|
|
11286
|
+
parseGraphqlEnvelope,
|
|
11287
|
+
graphqlInput,
|
|
11288
|
+
graphqlArgs,
|
|
11289
|
+
fetchGraphqlPage,
|
|
11290
|
+
fetchGraphqlList,
|
|
11291
|
+
operationPausedUntil,
|
|
11292
|
+
LIST_LIMIT,
|
|
11293
|
+
readGraphqlObserver,
|
|
11294
|
+
GRAPHQL_QUERIES,
|
|
11295
|
+
GRAPHQL_PAGE_SIZE,
|
|
11296
|
+
GRAPHQL_PAGE_POINTS,
|
|
11297
|
+
GRAPHQL_OBSERVER_POINTS,
|
|
9579
11298
|
mapAllSettledBounded,
|
|
9580
11299
|
alertRequestArgs,
|
|
9581
11300
|
shouldFetchAlertPriorityLanes,
|