gh-glance 0.9.1 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -1
- package/README.md +161 -76
- package/index.mjs +3400 -486
- package/package.json +2 -2
package/index.mjs
CHANGED
|
@@ -23,13 +23,14 @@
|
|
|
23
23
|
process.env.NODE_ENV ??= "production";
|
|
24
24
|
|
|
25
25
|
import { execFile, spawn } from "node:child_process";
|
|
26
|
-
import { createHash } from "node:crypto";
|
|
26
|
+
import { createHash, randomUUID } from "node:crypto";
|
|
27
27
|
import {
|
|
28
28
|
chmodSync,
|
|
29
29
|
closeSync,
|
|
30
30
|
mkdirSync,
|
|
31
31
|
openSync,
|
|
32
32
|
readFileSync,
|
|
33
|
+
readdirSync,
|
|
33
34
|
realpathSync,
|
|
34
35
|
renameSync,
|
|
35
36
|
statSync,
|
|
@@ -37,7 +38,7 @@ import {
|
|
|
37
38
|
writeFileSync,
|
|
38
39
|
} from "node:fs";
|
|
39
40
|
import { homedir } from "node:os";
|
|
40
|
-
import { dirname, isAbsolute, join } from "node:path";
|
|
41
|
+
import { basename, dirname, isAbsolute, join } from "node:path";
|
|
41
42
|
import { fileURLToPath } from "node:url";
|
|
42
43
|
import { promisify } from "node:util";
|
|
43
44
|
|
|
@@ -113,12 +114,11 @@ const ALERT_PER_PAGE = 100;
|
|
|
113
114
|
// parameters below.
|
|
114
115
|
const ALERT_QUERY = `?state=open&per_page=${ALERT_PER_PAGE}&sort=created&direction=desc`;
|
|
115
116
|
|
|
116
|
-
// Inactive tabs only feed the tab-bar counts
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
//
|
|
120
|
-
//
|
|
121
|
-
// other three exist to keep counts honest, which tolerates a minute.
|
|
117
|
+
// Inactive tabs only feed the tab-bar counts. One rotating background tab is
|
|
118
|
+
// considered every four active floors, so each inactive tab is considered
|
|
119
|
+
// about every Nth floor instead of all three starting together. The shared
|
|
120
|
+
// governor can schedule either active or background demand later when that is
|
|
121
|
+
// the safe resource slot.
|
|
122
122
|
const BACKGROUND_EVERY = 12;
|
|
123
123
|
// What one fetch of each tab costs lives in REST_PER_FETCH / GRAPHQL_PER_FETCH,
|
|
124
124
|
// declared with ALERT_SOURCES below because the security figure derives from it.
|
|
@@ -155,16 +155,6 @@ const BACKOFF_STEPS_MS = [60_000, 300_000, 1_800_000, 3_600_000];
|
|
|
155
155
|
// night costs -- two probes a minute per endpoint rather than one per tick.
|
|
156
156
|
const AUTH_RETRY_MS = [30_000];
|
|
157
157
|
|
|
158
|
-
// A rate limit is the third shape: it clears on GitHub's schedule, usually
|
|
159
|
-
// within the hour, and continuing to hammer is actively counterproductive --
|
|
160
|
-
// the secondary limiter keys on sustained request rate against a token that is
|
|
161
|
-
// already limited, so re-asking every tick can turn a self-clearing primary
|
|
162
|
-
// limit into a longer block, and the block applies to the *token*, not to this
|
|
163
|
-
// tool. A wedged pane could therefore degrade `git push` and everything else.
|
|
164
|
-
// One minute, flat: long enough to stop amplifying, short enough that the pane
|
|
165
|
-
// is current again well before a reset the user never notices.
|
|
166
|
-
const RATE_LIMIT_RETRY_MS = [60_000];
|
|
167
|
-
|
|
168
158
|
// Past this, the active tab's data is old enough to say so. Deliberately
|
|
169
159
|
// coarse: `now` only advances on minute boundaries when nothing is in progress,
|
|
170
160
|
// so a minute-granular staleness label costs zero extra redraws, whereas a
|
|
@@ -182,10 +172,12 @@ const runtime = {
|
|
|
182
172
|
// `gh api` calls get it as --hostname, because gh api has no --repo and would
|
|
183
173
|
// otherwise silently query github.com while the other tabs read the tenant.
|
|
184
174
|
host: null,
|
|
175
|
+
repoExplicit: false,
|
|
185
176
|
refreshMs: REFRESH_MS,
|
|
186
177
|
verbose: false,
|
|
187
178
|
initialTabIndex: 0,
|
|
188
179
|
};
|
|
180
|
+
let runtimeRemoteUrls = [];
|
|
189
181
|
|
|
190
182
|
// The four tab keys, needed by --tab validation which runs long before the TABS
|
|
191
183
|
// table itself is built.
|
|
@@ -611,7 +603,10 @@ function forwardSignalToChild(child, signal) {
|
|
|
611
603
|
return child.kill(signal);
|
|
612
604
|
}
|
|
613
605
|
|
|
614
|
-
async function runGh(args, { signal } = {}) {
|
|
606
|
+
async function runGh(args, { signal, operation } = {}) {
|
|
607
|
+
if (operationCost(operation) === null) {
|
|
608
|
+
throw new Error(`undeclared gh operation: ${operation ?? "missing"}`);
|
|
609
|
+
}
|
|
615
610
|
const startedAt = Date.now();
|
|
616
611
|
try {
|
|
617
612
|
const { stdout } = await execFileAsync("gh", args, {
|
|
@@ -633,7 +628,65 @@ async function runGh(args, { signal } = {}) {
|
|
|
633
628
|
// slug otherwise -- which is exactly the [HOST/]OWNER/REPO form `gh --repo`
|
|
634
629
|
// documents.
|
|
635
630
|
function qualifiedRepo() {
|
|
636
|
-
|
|
631
|
+
const host = runtime.repoExplicit ? effectiveRuntimeHost() : runtime.host;
|
|
632
|
+
return host ? `${host}/${runtime.repo}` : runtime.repo;
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
function normalizeHost(value) {
|
|
636
|
+
if (typeof value !== "string") return null;
|
|
637
|
+
const host = value.trim().toLowerCase();
|
|
638
|
+
return HOST_PATTERN.test(host) ? host : null;
|
|
639
|
+
}
|
|
640
|
+
|
|
641
|
+
function remoteHost(value) {
|
|
642
|
+
if (typeof value !== "string" || value.length === 0) return null;
|
|
643
|
+
try {
|
|
644
|
+
if (value.includes("://")) return normalizeHost(new URL(value).hostname);
|
|
645
|
+
} catch {
|
|
646
|
+
return null;
|
|
647
|
+
}
|
|
648
|
+
const scp = /^(?:[^@\s]+@)?([^:/\s]+):[^\s]+$/.exec(value);
|
|
649
|
+
return scp ? normalizeHost(scp[1]) : null;
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
// One resolver governs both the rate-limit route and the account coordinator.
|
|
653
|
+
// Explicit --repo is intentionally authoritative: a qualified value names its
|
|
654
|
+
// host, while an unqualified value means github.com even when GH_HOST or
|
|
655
|
+
// GH_REPO is also present. This is the approved Phase 2 precedence deviation.
|
|
656
|
+
function resolveEffectiveHost({
|
|
657
|
+
runtimeHost = null,
|
|
658
|
+
runtimeRepo = null,
|
|
659
|
+
repoExplicit = false,
|
|
660
|
+
ghHost = null,
|
|
661
|
+
ghRepo = null,
|
|
662
|
+
remoteUrls = [],
|
|
663
|
+
} = {}) {
|
|
664
|
+
if (repoExplicit && runtimeRepo) return normalizeHost(runtimeHost) ?? "github.com";
|
|
665
|
+
const explicitRuntimeHost = normalizeHost(runtimeHost);
|
|
666
|
+
if (explicitRuntimeHost) return explicitRuntimeHost;
|
|
667
|
+
if (ghHost !== null && ghHost !== undefined) return normalizeHost(ghHost);
|
|
668
|
+
if (ghRepo) {
|
|
669
|
+
try {
|
|
670
|
+
const target = parseRepoTarget(ghRepo);
|
|
671
|
+
return normalizeHost(target.host) ?? "github.com";
|
|
672
|
+
} catch {
|
|
673
|
+
return null;
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
const hosts = new Set(remoteUrls.map(remoteHost).filter(Boolean));
|
|
677
|
+
return hosts.size === 1 ? [...hosts][0] : null;
|
|
678
|
+
}
|
|
679
|
+
|
|
680
|
+
function effectiveRuntimeHost(options = {}) {
|
|
681
|
+
return resolveEffectiveHost({
|
|
682
|
+
runtimeHost: runtime.host,
|
|
683
|
+
runtimeRepo: runtime.repo,
|
|
684
|
+
repoExplicit: runtime.repoExplicit,
|
|
685
|
+
ghHost: process.env.GH_HOST,
|
|
686
|
+
ghRepo: process.env.GH_REPO,
|
|
687
|
+
remoteUrls: runtimeRemoteUrls,
|
|
688
|
+
...options,
|
|
689
|
+
});
|
|
637
690
|
}
|
|
638
691
|
|
|
639
692
|
// `--repo` for the list subcommands. Empty when unset, so the argv vector stays
|
|
@@ -648,8 +701,9 @@ function repoArgs() {
|
|
|
648
701
|
// list tabs and the alert endpoints disagree about which server they are
|
|
649
702
|
// talking to. --hostname is the flag gh api does have. Empty when no host was
|
|
650
703
|
// given, so the default argv vector is unchanged.
|
|
651
|
-
function apiHostArgs() {
|
|
652
|
-
|
|
704
|
+
function apiHostArgs(host = effectiveRuntimeHost()) {
|
|
705
|
+
host = normalizeHost(host);
|
|
706
|
+
return host ? ["--hostname", host] : [];
|
|
653
707
|
}
|
|
654
708
|
|
|
655
709
|
// `gh api` has no --repo; it resolves the {owner}/{repo} placeholder from the
|
|
@@ -757,10 +811,21 @@ function buildFailureContext(repoSettlement, authSettlement) {
|
|
|
757
811
|
return { repo, accounts };
|
|
758
812
|
}
|
|
759
813
|
|
|
760
|
-
async function resolveFailureContext(signal) {
|
|
814
|
+
async function resolveFailureContext(signal, governor = null, { run = runGh } = {}) {
|
|
815
|
+
const repoCall = governor
|
|
816
|
+
? runAdmittedOperation({
|
|
817
|
+
...governor,
|
|
818
|
+
operation: "failure-context:repository",
|
|
819
|
+
signal,
|
|
820
|
+
run: (admittedSignal) => run(repoContextArgs(), {
|
|
821
|
+
signal: admittedSignal,
|
|
822
|
+
operation: "failure-context:repository",
|
|
823
|
+
}),
|
|
824
|
+
}).then((result) => result.ok ? result.value : Promise.reject(result.error))
|
|
825
|
+
: Promise.reject(new Error("API budget unavailable"));
|
|
761
826
|
const [repo, auth] = await Promise.allSettled([
|
|
762
|
-
|
|
763
|
-
|
|
827
|
+
repoCall,
|
|
828
|
+
run(authContextArgs(), { signal, operation: "failure-context:auth" }),
|
|
764
829
|
]);
|
|
765
830
|
return buildFailureContext(repo, auth);
|
|
766
831
|
}
|
|
@@ -820,7 +885,7 @@ function actionsArgs(limit) {
|
|
|
820
885
|
}
|
|
821
886
|
|
|
822
887
|
async function fetchActions(limit, signal) {
|
|
823
|
-
const raw = await runGh(actionsArgs(limit), { signal });
|
|
888
|
+
const raw = await runGh(actionsArgs(limit), { signal, operation: "tab:actions" });
|
|
824
889
|
return {
|
|
825
890
|
raw,
|
|
826
891
|
limit,
|
|
@@ -870,7 +935,7 @@ function issuesArgs() {
|
|
|
870
935
|
}
|
|
871
936
|
|
|
872
937
|
async function fetchIssues(signal) {
|
|
873
|
-
const raw = await runGh(issuesArgs(), { signal });
|
|
938
|
+
const raw = await runGh(issuesArgs(), { signal, operation: "tab:issues" });
|
|
874
939
|
return {
|
|
875
940
|
raw,
|
|
876
941
|
limit: LIST_LIMIT,
|
|
@@ -905,7 +970,7 @@ function prsArgs() {
|
|
|
905
970
|
}
|
|
906
971
|
|
|
907
972
|
async function fetchPRs(signal) {
|
|
908
|
-
const raw = await runGh(prsArgs(), { signal });
|
|
973
|
+
const raw = await runGh(prsArgs(), { signal, operation: "tab:prs" });
|
|
909
974
|
return {
|
|
910
975
|
raw,
|
|
911
976
|
limit: LIST_LIMIT,
|
|
@@ -992,146 +1057,1641 @@ const ALERT_SOURCES = [
|
|
|
992
1057
|
},
|
|
993
1058
|
];
|
|
994
1059
|
|
|
995
|
-
// What one fetch of each tab costs, per budget. The single source of truth:
|
|
996
|
-
// `projectedHourlyCost` derives the hourly figure from it and the spend meter
|
|
997
|
-
// bills against it, so a corrected number cannot reach one and miss the other.
|
|
998
|
-
//
|
|
999
|
-
// actions is 2, not 1: measured 2026-08-10 with `GH_DEBUG=api`, `gh run list`
|
|
1000
|
-
// issues GET /actions/runs *and* GET /actions/workflows. The 1 that stood here
|
|
1001
|
-
// understated the default tab -- the tab every pane starts on, since
|
|
1002
|
-
// initialTabIndex defaults to 0 -- by half.
|
|
1003
|
-
//
|
|
1004
|
-
// issues and prs are 0 REST because SORT_RECENT's --search routes both through
|
|
1005
|
-
// GraphQL entirely (2 POSTs each, confirmed by the same measurement).
|
|
1006
|
-
const SECURITY_REQUESTS_PER_FETCH = ALERT_SOURCES.reduce(
|
|
1007
|
-
(total, source) => total + 1 + source.priorityQueries.length,
|
|
1008
|
-
0,
|
|
1009
|
-
);
|
|
1010
|
-
const REST_PER_FETCH = { actions: 2, issues: 0, prs: 0, security: SECURITY_REQUESTS_PER_FETCH };
|
|
1011
|
-
const GRAPHQL_PER_FETCH = { actions: 0, issues: 2, prs: 2, security: 0 };
|
|
1060
|
+
// What one fetch of each tab costs, per budget. The single source of truth:
|
|
1061
|
+
// `projectedHourlyCost` derives the hourly figure from it and the spend meter
|
|
1062
|
+
// bills against it, so a corrected number cannot reach one and miss the other.
|
|
1063
|
+
//
|
|
1064
|
+
// actions is 2, not 1: measured 2026-08-10 with `GH_DEBUG=api`, `gh run list`
|
|
1065
|
+
// issues GET /actions/runs *and* GET /actions/workflows. The 1 that stood here
|
|
1066
|
+
// understated the default tab -- the tab every pane starts on, since
|
|
1067
|
+
// initialTabIndex defaults to 0 -- by half.
|
|
1068
|
+
//
|
|
1069
|
+
// issues and prs are 0 REST because SORT_RECENT's --search routes both through
|
|
1070
|
+
// GraphQL entirely (2 POSTs each, confirmed by the same measurement).
|
|
1071
|
+
const SECURITY_REQUESTS_PER_FETCH = ALERT_SOURCES.reduce(
|
|
1072
|
+
(total, source) => total + 1 + source.priorityQueries.length,
|
|
1073
|
+
0,
|
|
1074
|
+
);
|
|
1075
|
+
const REST_PER_FETCH = { actions: 2, issues: 0, prs: 0, security: SECURITY_REQUESTS_PER_FETCH };
|
|
1076
|
+
const GRAPHQL_PER_FETCH = { actions: 0, issues: 2, prs: 2, security: 0 };
|
|
1077
|
+
|
|
1078
|
+
function tabRequestCost(tab) {
|
|
1079
|
+
if (!TAB_KEYS.includes(tab)) return null;
|
|
1080
|
+
return { core: REST_PER_FETCH[tab], graphql: GRAPHQL_PER_FETCH[tab] };
|
|
1081
|
+
}
|
|
1082
|
+
|
|
1083
|
+
// Every gh subprocess has one declared operation, enforced at the subprocess
|
|
1084
|
+
// boundary. The shared governor uses these costs for quota admission. Tab totals are
|
|
1085
|
+
// derived from the fetch tables, while Security endpoint entries describe the
|
|
1086
|
+
// individual calls covered by the tab's six-call upfront reservation.
|
|
1087
|
+
const OPERATION_COSTS = Object.freeze({
|
|
1088
|
+
...Object.fromEntries(TAB_KEYS.map((tab) => [`tab:${tab}`, tabRequestCost(tab)])),
|
|
1089
|
+
"tab:security-endpoint": { core: 1, graphql: 0 },
|
|
1090
|
+
"failure-context:repository": { core: 0, graphql: 1 },
|
|
1091
|
+
"failure-context:auth": { core: 0, graphql: 0 },
|
|
1092
|
+
"open:actions": { core: REST_PER_FETCH.actions, graphql: 0 },
|
|
1093
|
+
"open:issues": { core: 0, graphql: GRAPHQL_PER_FETCH.issues },
|
|
1094
|
+
"open:prs": { core: 0, graphql: GRAPHQL_PER_FETCH.prs },
|
|
1095
|
+
"doctor:repository": { core: 0, graphql: 1 },
|
|
1096
|
+
"doctor:actions": tabRequestCost("actions"),
|
|
1097
|
+
"doctor:issues": tabRequestCost("issues"),
|
|
1098
|
+
"doctor:prs": tabRequestCost("prs"),
|
|
1099
|
+
"doctor:security-endpoint": { core: 1, graphql: 0 },
|
|
1100
|
+
"rate-limit": { core: 0, graphql: 0 },
|
|
1101
|
+
"version": { core: 0, graphql: 0 },
|
|
1102
|
+
"auth-status": { core: 0, graphql: 0 },
|
|
1103
|
+
"local-git": { core: 0, graphql: 0 },
|
|
1104
|
+
});
|
|
1105
|
+
|
|
1106
|
+
function operationCost(operation) {
|
|
1107
|
+
return pick(OPERATION_COSTS, operation, null);
|
|
1108
|
+
}
|
|
1109
|
+
|
|
1110
|
+
// Target this fraction of what the token can afford, not all of it. The margin
|
|
1111
|
+
// is for the user's own `gh` and `git` commands.
|
|
1112
|
+
const BUDGET_SAFETY = 0.8;
|
|
1113
|
+
const BUDGET_RESERVE_FRACTION = 1 - BUDGET_SAFETY;
|
|
1114
|
+
const BUDGET_SNAPSHOT_TTL_MS = 65_000;
|
|
1115
|
+
const GOVERNOR_HEARTBEAT_MS = 20_000;
|
|
1116
|
+
const GOVERNOR_LEASE_TTL_MS = 90_000;
|
|
1117
|
+
const GOVERNOR_PROBE_LEASE_MS = 70_000;
|
|
1118
|
+
const GOVERNOR_ACTIVE_PROBE_LEASE_MS = 35_000;
|
|
1119
|
+
const BUDGET_RESET_GRACE_MS = 2_000;
|
|
1120
|
+
const GOVERNOR_PHASE_WINDOW_MS = 5_000;
|
|
1121
|
+
const BUDGET_WINDOW_MS = 3_600_000;
|
|
1122
|
+
|
|
1123
|
+
// How often to re-read the budget. `gh api rate_limit` does not count against
|
|
1124
|
+
// the limit (verified, delta 0 -- see rateBudget) but it is still a subprocess,
|
|
1125
|
+
// so once a minute rather than once a tick: the quantity it measures moves on
|
|
1126
|
+
// the scale of minutes.
|
|
1127
|
+
const BUDGET_PROBE_MS = 60_000;
|
|
1128
|
+
|
|
1129
|
+
// Below this many completed shared calls in a probe window, external-spend
|
|
1130
|
+
// inference is noise. Under the threshold the loop retains the previous factor.
|
|
1131
|
+
const MIN_SAMPLE_CALLS = 5;
|
|
1132
|
+
|
|
1133
|
+
const REQUEST_PRIORITIES = Object.freeze({
|
|
1134
|
+
manual: 0,
|
|
1135
|
+
diagnostic: 0,
|
|
1136
|
+
"tab-switch": 1,
|
|
1137
|
+
active: 2,
|
|
1138
|
+
background: 3,
|
|
1139
|
+
});
|
|
1140
|
+
|
|
1141
|
+
function normalizeBudgetResource(raw, observedAt = raw?.observedAt) {
|
|
1142
|
+
if (!raw || typeof raw !== "object") return null;
|
|
1143
|
+
const normalized = {
|
|
1144
|
+
limit: raw.limit,
|
|
1145
|
+
remaining: raw.remaining,
|
|
1146
|
+
used: raw.used,
|
|
1147
|
+
resetMs: raw.resetMs,
|
|
1148
|
+
observedAt,
|
|
1149
|
+
};
|
|
1150
|
+
if (
|
|
1151
|
+
Object.values(normalized).some((value) => !Number.isFinite(value) || value < 0) ||
|
|
1152
|
+
normalized.remaining > normalized.limit ||
|
|
1153
|
+
normalized.used > normalized.limit
|
|
1154
|
+
) {
|
|
1155
|
+
return null;
|
|
1156
|
+
}
|
|
1157
|
+
return normalized;
|
|
1158
|
+
}
|
|
1159
|
+
|
|
1160
|
+
function budgetEpoch(resource) {
|
|
1161
|
+
const normalized = normalizeBudgetResource(resource);
|
|
1162
|
+
if (!normalized) return null;
|
|
1163
|
+
return typeof resource?.epoch === "string" && resource.epoch.length > 0
|
|
1164
|
+
? resource.epoch
|
|
1165
|
+
: `${normalized.limit}:${normalized.resetMs}`;
|
|
1166
|
+
}
|
|
1167
|
+
|
|
1168
|
+
function resourceReserve(limit) {
|
|
1169
|
+
return Number.isFinite(limit) && limit >= 0
|
|
1170
|
+
? Math.ceil(limit * BUDGET_RESERVE_FRACTION)
|
|
1171
|
+
: null;
|
|
1172
|
+
}
|
|
1173
|
+
|
|
1174
|
+
function leaseFor(leases, leaseId) {
|
|
1175
|
+
return leases instanceof Map ? leases.get(leaseId) : leases?.[leaseId];
|
|
1176
|
+
}
|
|
1177
|
+
|
|
1178
|
+
function reservationCost(reservation, resource, leases, nowMs) {
|
|
1179
|
+
if (!reservation || ["cancelled", "reconciled"].includes(reservation.status)) return 0;
|
|
1180
|
+
if (reservation.status === "scheduled") {
|
|
1181
|
+
const lease = leaseFor(leases, reservation.leaseId);
|
|
1182
|
+
if (!lease || (Number.isFinite(lease.expiresAt) && lease.expiresAt <= nowMs)) return 0;
|
|
1183
|
+
}
|
|
1184
|
+
const chargedCosts = reservation.status === "completed" &&
|
|
1185
|
+
reservation.outcome === "measured-success" && reservation.actualCosts
|
|
1186
|
+
? reservation.actualCosts
|
|
1187
|
+
: reservation.costs;
|
|
1188
|
+
const cost = chargedCosts?.[resource] ??
|
|
1189
|
+
(reservation.resource === resource ? reservation.cost : 0);
|
|
1190
|
+
return Number.isFinite(cost) && cost > 0 ? cost : 0;
|
|
1191
|
+
}
|
|
1192
|
+
|
|
1193
|
+
function availableForGrant({
|
|
1194
|
+
budget,
|
|
1195
|
+
reservations = [],
|
|
1196
|
+
leases = {},
|
|
1197
|
+
nowMs,
|
|
1198
|
+
chargedCost = null,
|
|
1199
|
+
}) {
|
|
1200
|
+
const normalized = normalizeBudgetResource(budget);
|
|
1201
|
+
if (!normalized) return { mode: "probe", reason: "budget-unknown" };
|
|
1202
|
+
if (normalized.observedAt > nowMs) return { mode: "probe", reason: "budget-future" };
|
|
1203
|
+
if (nowMs - normalized.observedAt > BUDGET_SNAPSHOT_TTL_MS) {
|
|
1204
|
+
return { mode: "probe", reason: "budget-stale" };
|
|
1205
|
+
}
|
|
1206
|
+
if (nowMs >= normalized.resetMs) return { mode: "probe", reason: "budget-reset" };
|
|
1207
|
+
|
|
1208
|
+
const epoch = budgetEpoch(budget);
|
|
1209
|
+
if (Number.isFinite(budget.blockUntil) && budget.blockUntil > nowMs) {
|
|
1210
|
+
return {
|
|
1211
|
+
mode: "paused",
|
|
1212
|
+
reason: budget.blockReason ?? "rate-limit",
|
|
1213
|
+
resetMs: normalized.resetMs,
|
|
1214
|
+
epoch,
|
|
1215
|
+
};
|
|
1216
|
+
}
|
|
1217
|
+
|
|
1218
|
+
if (chargedCost !== null && (!Number.isFinite(chargedCost) || chargedCost < 0)) {
|
|
1219
|
+
return {
|
|
1220
|
+
mode: "paused",
|
|
1221
|
+
reason: "reservations-invalid",
|
|
1222
|
+
resetMs: normalized.resetMs,
|
|
1223
|
+
epoch,
|
|
1224
|
+
};
|
|
1225
|
+
}
|
|
1226
|
+
|
|
1227
|
+
const reserve = resourceReserve(normalized.limit);
|
|
1228
|
+
const charged = chargedCost ?? reservations.reduce(
|
|
1229
|
+
(total, reservation) => total + reservationCost(reservation, budget.resource, leases, nowMs),
|
|
1230
|
+
0,
|
|
1231
|
+
);
|
|
1232
|
+
return {
|
|
1233
|
+
mode: "open",
|
|
1234
|
+
reserve,
|
|
1235
|
+
spendable: Math.max(0, normalized.remaining - reserve - charged),
|
|
1236
|
+
resetMs: normalized.resetMs,
|
|
1237
|
+
epoch,
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
|
|
1241
|
+
function nextExternalFactor({
|
|
1242
|
+
lastExternalFactor = 1,
|
|
1243
|
+
globalUsedDelta = 0,
|
|
1244
|
+
sharedCompletedDelta = 0,
|
|
1245
|
+
}) {
|
|
1246
|
+
if (
|
|
1247
|
+
!Number.isFinite(lastExternalFactor) ||
|
|
1248
|
+
lastExternalFactor < 1 ||
|
|
1249
|
+
!Number.isFinite(globalUsedDelta) ||
|
|
1250
|
+
globalUsedDelta < 0 ||
|
|
1251
|
+
!Number.isFinite(sharedCompletedDelta) ||
|
|
1252
|
+
sharedCompletedDelta < 0
|
|
1253
|
+
) {
|
|
1254
|
+
return null;
|
|
1255
|
+
}
|
|
1256
|
+
const sample = { globalUsedDelta, sharedCompletedDelta };
|
|
1257
|
+
if (!externalSampleIsUsable(sample)) return lastExternalFactor;
|
|
1258
|
+
const measured = sample.globalUsedDelta / sample.sharedCompletedDelta;
|
|
1259
|
+
return Number.isFinite(measured) ? Math.max(1, measured) : null;
|
|
1260
|
+
}
|
|
1261
|
+
|
|
1262
|
+
function resourceDecision({
|
|
1263
|
+
budget,
|
|
1264
|
+
resource,
|
|
1265
|
+
reservations = [],
|
|
1266
|
+
leases = {},
|
|
1267
|
+
nowMs,
|
|
1268
|
+
cost = 0,
|
|
1269
|
+
lastExternalFactor = budget?.lastExternalFactor ?? 1,
|
|
1270
|
+
globalUsedDelta = 0,
|
|
1271
|
+
sharedCompletedDelta = 0,
|
|
1272
|
+
chargedCost = null,
|
|
1273
|
+
}) {
|
|
1274
|
+
const capacity = availableForGrant({
|
|
1275
|
+
budget: budget && { ...budget, resource },
|
|
1276
|
+
reservations,
|
|
1277
|
+
leases,
|
|
1278
|
+
nowMs,
|
|
1279
|
+
chargedCost,
|
|
1280
|
+
});
|
|
1281
|
+
if (capacity.mode !== "open") return capacity;
|
|
1282
|
+
|
|
1283
|
+
const externalFactor = nextExternalFactor({
|
|
1284
|
+
lastExternalFactor,
|
|
1285
|
+
globalUsedDelta,
|
|
1286
|
+
sharedCompletedDelta,
|
|
1287
|
+
});
|
|
1288
|
+
if (externalFactor === null) {
|
|
1289
|
+
return {
|
|
1290
|
+
mode: "paused",
|
|
1291
|
+
reason: "external-factor-invalid",
|
|
1292
|
+
resetMs: capacity.resetMs,
|
|
1293
|
+
epoch: capacity.epoch,
|
|
1294
|
+
};
|
|
1295
|
+
}
|
|
1296
|
+
if (!Number.isFinite(cost) || cost < 0 || capacity.spendable < cost || capacity.spendable <= 0) {
|
|
1297
|
+
return {
|
|
1298
|
+
mode: "paused",
|
|
1299
|
+
reason: "reserve",
|
|
1300
|
+
resetMs: capacity.resetMs,
|
|
1301
|
+
epoch: capacity.epoch,
|
|
1302
|
+
};
|
|
1303
|
+
}
|
|
1304
|
+
const callsPerMs = capacity.spendable / (capacity.resetMs - nowMs) / externalFactor;
|
|
1305
|
+
if (!Number.isFinite(callsPerMs) || callsPerMs <= 0) {
|
|
1306
|
+
return {
|
|
1307
|
+
mode: "paused",
|
|
1308
|
+
reason: "pacing-invalid",
|
|
1309
|
+
resetMs: capacity.resetMs,
|
|
1310
|
+
epoch: capacity.epoch,
|
|
1311
|
+
};
|
|
1312
|
+
}
|
|
1313
|
+
return { ...capacity, callsPerMs, externalFactor };
|
|
1314
|
+
}
|
|
1315
|
+
|
|
1316
|
+
function governorPhaseOffset(phaseSeed, epoch) {
|
|
1317
|
+
const text = `${phaseSeed}:${epoch}`;
|
|
1318
|
+
let hash = 2166136261;
|
|
1319
|
+
for (let index = 0; index < text.length; index += 1) {
|
|
1320
|
+
hash ^= text.charCodeAt(index);
|
|
1321
|
+
hash = Math.imul(hash, 16777619);
|
|
1322
|
+
}
|
|
1323
|
+
return (hash >>> 0) % (GOVERNOR_PHASE_WINDOW_MS + 1);
|
|
1324
|
+
}
|
|
1325
|
+
|
|
1326
|
+
function governorEpochPhaseAt(phaseSeed, decision) {
|
|
1327
|
+
const epochAnchor = Math.max(phaseSeed.registeredAt, decision.resetMs - BUDGET_WINDOW_MS);
|
|
1328
|
+
return epochAnchor + governorPhaseOffset(phaseSeed.seed, decision.epoch);
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
function intentPriority(intent) {
|
|
1332
|
+
const priority = Number.isInteger(intent.priority)
|
|
1333
|
+
? intent.priority
|
|
1334
|
+
: REQUEST_PRIORITIES[intent.priority];
|
|
1335
|
+
return Object.values(REQUEST_PRIORITIES).includes(priority) ? priority : null;
|
|
1336
|
+
}
|
|
1337
|
+
|
|
1338
|
+
const RATE_RESOURCES = ["core", "graphql"];
|
|
1339
|
+
|
|
1340
|
+
function exactResourceCosts(raw) {
|
|
1341
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
|
|
1342
|
+
const keys = Object.keys(raw);
|
|
1343
|
+
if (
|
|
1344
|
+
keys.length !== RATE_RESOURCES.length ||
|
|
1345
|
+
RATE_RESOURCES.some(
|
|
1346
|
+
(resource) =>
|
|
1347
|
+
!Object.hasOwn(raw, resource) || !Number.isFinite(raw[resource]) || raw[resource] < 0,
|
|
1348
|
+
)
|
|
1349
|
+
) {
|
|
1350
|
+
return null;
|
|
1351
|
+
}
|
|
1352
|
+
return { core: raw.core, graphql: raw.graphql };
|
|
1353
|
+
}
|
|
1354
|
+
|
|
1355
|
+
function intentCosts(intent) {
|
|
1356
|
+
if (intent.tab === undefined) return exactResourceCosts(intent.costs);
|
|
1357
|
+
const expected = tabRequestCost(intent.tab);
|
|
1358
|
+
if (!expected) return null;
|
|
1359
|
+
const supplied = intent.costs === undefined ? expected : exactResourceCosts(intent.costs);
|
|
1360
|
+
if (!supplied || supplied.core !== expected.core || supplied.graphql !== expected.graphql) {
|
|
1361
|
+
return null;
|
|
1362
|
+
}
|
|
1363
|
+
return expected;
|
|
1364
|
+
}
|
|
1365
|
+
|
|
1366
|
+
function normalizePhaseSeed(raw) {
|
|
1367
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
|
|
1368
|
+
const validSeed =
|
|
1369
|
+
typeof raw.seed === "string" ? raw.seed.length > 0 : Number.isFinite(raw.seed);
|
|
1370
|
+
return validSeed && Number.isFinite(raw.registeredAt) && raw.registeredAt >= 0
|
|
1371
|
+
? { seed: raw.seed, registeredAt: raw.registeredAt }
|
|
1372
|
+
: null;
|
|
1373
|
+
}
|
|
1374
|
+
|
|
1375
|
+
function createRoundRobinState(intents, cursors) {
|
|
1376
|
+
const state = {};
|
|
1377
|
+
for (const resource of RATE_RESOURCES) {
|
|
1378
|
+
const leaseIds = new Set(cursors[resource] ? [cursors[resource]] : []);
|
|
1379
|
+
for (const intent of intents) {
|
|
1380
|
+
if (intent.costs[resource] > 0) leaseIds.add(intent.leaseId);
|
|
1381
|
+
}
|
|
1382
|
+
const order = [...leaseIds].sort();
|
|
1383
|
+
const index = new Map(order.map((leaseId, position) => [leaseId, position]));
|
|
1384
|
+
const cursorIndex = index.get(cursors[resource]);
|
|
1385
|
+
state[resource] = {
|
|
1386
|
+
index,
|
|
1387
|
+
size: order.length,
|
|
1388
|
+
start: cursorIndex === undefined || order.length === 0
|
|
1389
|
+
? 0
|
|
1390
|
+
: (cursorIndex + 1) % order.length,
|
|
1391
|
+
};
|
|
1392
|
+
}
|
|
1393
|
+
state.multiplier = intents.length * RATE_RESOURCES.length + 1;
|
|
1394
|
+
return state;
|
|
1395
|
+
}
|
|
1396
|
+
|
|
1397
|
+
function fairnessScore(intent, state) {
|
|
1398
|
+
let maximum = 0;
|
|
1399
|
+
let total = 0;
|
|
1400
|
+
for (const resource of RATE_RESOURCES) {
|
|
1401
|
+
if (intent.costs[resource] <= 0) continue;
|
|
1402
|
+
const lane = state[resource];
|
|
1403
|
+
const position = lane.index.get(intent.leaseId);
|
|
1404
|
+
const distance = (position - lane.start + lane.size) % lane.size;
|
|
1405
|
+
maximum = Math.max(maximum, distance);
|
|
1406
|
+
total += distance;
|
|
1407
|
+
}
|
|
1408
|
+
return maximum * state.multiplier + total;
|
|
1409
|
+
}
|
|
1410
|
+
|
|
1411
|
+
function nextRoundRobinIndex(intents, state) {
|
|
1412
|
+
let bestIndex = 0;
|
|
1413
|
+
let bestScore = fairnessScore(intents[0], state);
|
|
1414
|
+
for (let index = 1; index < intents.length; index += 1) {
|
|
1415
|
+
const score = fairnessScore(intents[index], state);
|
|
1416
|
+
const best = intents[bestIndex];
|
|
1417
|
+
const candidate = intents[index];
|
|
1418
|
+
if (
|
|
1419
|
+
score < bestScore ||
|
|
1420
|
+
(score === bestScore &&
|
|
1421
|
+
((candidate.requestedAt ?? 0) < (best.requestedAt ?? 0) ||
|
|
1422
|
+
((candidate.requestedAt ?? 0) === (best.requestedAt ?? 0) &&
|
|
1423
|
+
String(candidate.id).localeCompare(String(best.id)) < 0)))
|
|
1424
|
+
) {
|
|
1425
|
+
bestIndex = index;
|
|
1426
|
+
bestScore = score;
|
|
1427
|
+
}
|
|
1428
|
+
}
|
|
1429
|
+
return bestIndex;
|
|
1430
|
+
}
|
|
1431
|
+
|
|
1432
|
+
function advanceRoundRobinState(state, resource, leaseId) {
|
|
1433
|
+
const lane = state[resource];
|
|
1434
|
+
lane.start = (lane.index.get(leaseId) + 1) % lane.size;
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
function scheduleIntents({
|
|
1438
|
+
intents = [],
|
|
1439
|
+
leases = {},
|
|
1440
|
+
budgets = {},
|
|
1441
|
+
reservations = [],
|
|
1442
|
+
lanes = {},
|
|
1443
|
+
cursors = {},
|
|
1444
|
+
nowMs,
|
|
1445
|
+
maxGrants = Number.POSITIVE_INFINITY,
|
|
1446
|
+
}) {
|
|
1447
|
+
const valid = [];
|
|
1448
|
+
const prunedIntentIds = [];
|
|
1449
|
+
for (const intent of intents) {
|
|
1450
|
+
const lease = leaseFor(leases, intent.leaseId);
|
|
1451
|
+
const costs = intentCosts(intent);
|
|
1452
|
+
const priority = intentPriority(intent);
|
|
1453
|
+
const phaseSeed = normalizePhaseSeed(lease?.phaseSeed);
|
|
1454
|
+
const knownTab = intent.tab === undefined || TAB_KEYS.includes(intent.tab);
|
|
1455
|
+
if (
|
|
1456
|
+
!lease ||
|
|
1457
|
+
!Number.isFinite(lease.expiresAt) ||
|
|
1458
|
+
lease.expiresAt <= nowMs ||
|
|
1459
|
+
!Number.isFinite(intent.expiresAt) ||
|
|
1460
|
+
intent.expiresAt <= nowMs ||
|
|
1461
|
+
!phaseSeed ||
|
|
1462
|
+
!knownTab ||
|
|
1463
|
+
priority === null ||
|
|
1464
|
+
!costs
|
|
1465
|
+
) {
|
|
1466
|
+
prunedIntentIds.push(intent.id);
|
|
1467
|
+
} else {
|
|
1468
|
+
valid.push({ ...intent, costs, phaseSeed, normalizedPriority: priority });
|
|
1469
|
+
}
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
const chargedTotals = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
1473
|
+
resource,
|
|
1474
|
+
reservations.reduce(
|
|
1475
|
+
(total, reservation) => total + reservationCost(reservation, resource, leases, nowMs),
|
|
1476
|
+
0,
|
|
1477
|
+
),
|
|
1478
|
+
]));
|
|
1479
|
+
const updatedLanes = structuredClone(lanes);
|
|
1480
|
+
const updatedCursors = { ...cursors };
|
|
1481
|
+
const grants = [];
|
|
1482
|
+
const denied = [];
|
|
1483
|
+
const priorities = [...new Set(valid.map((intent) => intent.normalizedPriority))].sort((a, b) => a - b);
|
|
1484
|
+
|
|
1485
|
+
for (const priority of priorities) {
|
|
1486
|
+
const pending = valid.filter((intent) => intent.normalizedPriority === priority);
|
|
1487
|
+
const roundRobinState = createRoundRobinState(pending, updatedCursors);
|
|
1488
|
+
while (pending.length > 0) {
|
|
1489
|
+
if (grants.length >= maxGrants) break;
|
|
1490
|
+
const intent = pending.splice(nextRoundRobinIndex(pending, roundRobinState), 1)[0];
|
|
1491
|
+
const resources = RATE_RESOURCES.filter((resource) => intent.costs[resource] > 0);
|
|
1492
|
+
const decisions = Object.fromEntries(resources.map((resource) => [
|
|
1493
|
+
resource,
|
|
1494
|
+
resourceDecision({
|
|
1495
|
+
budget: budgets[resource],
|
|
1496
|
+
resource,
|
|
1497
|
+
leases,
|
|
1498
|
+
nowMs,
|
|
1499
|
+
cost: intent.costs[resource],
|
|
1500
|
+
chargedCost: chargedTotals[resource],
|
|
1501
|
+
}),
|
|
1502
|
+
]));
|
|
1503
|
+
const blocked = resources.find((resource) => decisions[resource].mode !== "open");
|
|
1504
|
+
if (blocked) {
|
|
1505
|
+
denied.push({ intentId: intent.id, resource: blocked, ...decisions[blocked] });
|
|
1506
|
+
continue;
|
|
1507
|
+
}
|
|
1508
|
+
|
|
1509
|
+
const phaseTimes = Object.fromEntries(resources.map((resource) => [
|
|
1510
|
+
resource,
|
|
1511
|
+
governorEpochPhaseAt(intent.phaseSeed, decisions[resource]),
|
|
1512
|
+
]));
|
|
1513
|
+
const notBefore = Math.max(
|
|
1514
|
+
nowMs,
|
|
1515
|
+
...Object.values(phaseTimes),
|
|
1516
|
+
...resources.map((resource) => updatedLanes[resource]?.nextAt ?? nowMs),
|
|
1517
|
+
);
|
|
1518
|
+
const expiring = resources.find((resource) => notBefore >= decisions[resource].resetMs);
|
|
1519
|
+
if (expiring) {
|
|
1520
|
+
denied.push({
|
|
1521
|
+
intentId: intent.id,
|
|
1522
|
+
resource: expiring,
|
|
1523
|
+
mode: "waiting",
|
|
1524
|
+
reason: "reset",
|
|
1525
|
+
retryAt: decisions[expiring].resetMs + BUDGET_RESET_GRACE_MS,
|
|
1526
|
+
resetMs: decisions[expiring].resetMs,
|
|
1527
|
+
epoch: decisions[expiring].epoch,
|
|
1528
|
+
});
|
|
1529
|
+
continue;
|
|
1530
|
+
}
|
|
1531
|
+
|
|
1532
|
+
const reservation = {
|
|
1533
|
+
id: `reservation:${intent.id}`,
|
|
1534
|
+
intentId: intent.id,
|
|
1535
|
+
leaseId: intent.leaseId,
|
|
1536
|
+
costs: { ...intent.costs },
|
|
1537
|
+
notBefore,
|
|
1538
|
+
status: "scheduled",
|
|
1539
|
+
epochs: Object.fromEntries(resources.map((resource) => [resource, decisions[resource].epoch])),
|
|
1540
|
+
};
|
|
1541
|
+
grants.push(reservation);
|
|
1542
|
+
for (const resource of resources) {
|
|
1543
|
+
chargedTotals[resource] += intent.costs[resource];
|
|
1544
|
+
updatedLanes[resource] = {
|
|
1545
|
+
...updatedLanes[resource],
|
|
1546
|
+
nextAt: notBefore + intent.costs[resource] / decisions[resource].callsPerMs,
|
|
1547
|
+
};
|
|
1548
|
+
updatedCursors[resource] = intent.leaseId;
|
|
1549
|
+
advanceRoundRobinState(roundRobinState, resource, intent.leaseId);
|
|
1550
|
+
}
|
|
1551
|
+
}
|
|
1552
|
+
}
|
|
1553
|
+
|
|
1554
|
+
return {
|
|
1555
|
+
grants,
|
|
1556
|
+
lanes: updatedLanes,
|
|
1557
|
+
cursors: updatedCursors,
|
|
1558
|
+
denied,
|
|
1559
|
+
prunedIntentIds,
|
|
1560
|
+
};
|
|
1561
|
+
}
|
|
1562
|
+
|
|
1563
|
+
// ---------- Shared account governor ----------
|
|
1564
|
+
|
|
1565
|
+
const GOVERNOR_STATE_VERSION = 1;
|
|
1566
|
+
const GOVERNOR_MAX_LEASES = 128;
|
|
1567
|
+
const GOVERNOR_MAX_INTENTS = 512;
|
|
1568
|
+
const GOVERNOR_MAX_RESERVATIONS = 512;
|
|
1569
|
+
const GOVERNOR_LOCK_WAIT_MS = 250;
|
|
1570
|
+
const GOVERNOR_PROBE_DRAIN_MS = 30_000;
|
|
1571
|
+
const GOVERNOR_PUBLICATION_REINSPECT_MS = 1_000;
|
|
1572
|
+
const GOVERNOR_PROBE_TRANSITION_MS = 5_000;
|
|
1573
|
+
const GOVERNOR_MAX_FUTURE_MS = 24 * 60 * 60 * 1000;
|
|
1574
|
+
const GOVERNOR_OUTCOMES = new Set([
|
|
1575
|
+
"measured-success",
|
|
1576
|
+
"rejected",
|
|
1577
|
+
"timeout",
|
|
1578
|
+
"signal",
|
|
1579
|
+
"abort",
|
|
1580
|
+
"process-loss",
|
|
1581
|
+
]);
|
|
1582
|
+
const GOVERNOR_RESERVATION_STATUSES = new Set([
|
|
1583
|
+
"scheduled",
|
|
1584
|
+
"started",
|
|
1585
|
+
"completed",
|
|
1586
|
+
"cancelled",
|
|
1587
|
+
]);
|
|
1588
|
+
const GOVERNOR_PROBE_STATUSES = new Set(["idle", "waiting", "healthy", "failed"]);
|
|
1589
|
+
const GOVERNOR_BLOCK_REASONS = new Set(["rate-limit", "secondary-rate-limit", "abuse-limit"]);
|
|
1590
|
+
const GOVERNOR_ID_PATTERN = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
1591
|
+
|
|
1592
|
+
function governorId() {
|
|
1593
|
+
return randomUUID();
|
|
1594
|
+
}
|
|
1595
|
+
|
|
1596
|
+
function validGovernorId(value) {
|
|
1597
|
+
return typeof value === "string" && GOVERNOR_ID_PATTERN.test(value);
|
|
1598
|
+
}
|
|
1599
|
+
|
|
1600
|
+
function validGovernorEpoch(value) {
|
|
1601
|
+
return typeof value === "string" && /^[0-9:.]+$/.test(value);
|
|
1602
|
+
}
|
|
1603
|
+
|
|
1604
|
+
function governorScopeHash(effectiveHost, authIdentity) {
|
|
1605
|
+
const host = normalizeHost(effectiveHost);
|
|
1606
|
+
if (!host || typeof authIdentity !== "string" || authIdentity.length === 0) return null;
|
|
1607
|
+
return createHash("sha256")
|
|
1608
|
+
.update(JSON.stringify({ version: GOVERNOR_STATE_VERSION, effectiveHost: host, authCacheIdentity: authIdentity }))
|
|
1609
|
+
.digest("hex");
|
|
1610
|
+
}
|
|
1611
|
+
|
|
1612
|
+
function governorPath(scopeHash, options = {}) {
|
|
1613
|
+
return join(dirname(widthPreferencesPath(options)), `rate-governor-v1-${scopeHash}.json`);
|
|
1614
|
+
}
|
|
1615
|
+
|
|
1616
|
+
function createGovernorScope({
|
|
1617
|
+
effectiveHost,
|
|
1618
|
+
authIdentity,
|
|
1619
|
+
identityProvider = null,
|
|
1620
|
+
now = Date.now,
|
|
1621
|
+
kill = process.kill.bind(process),
|
|
1622
|
+
...pathOptions
|
|
1623
|
+
} = {}) {
|
|
1624
|
+
const host = normalizeHost(effectiveHost);
|
|
1625
|
+
const resolvedAuthIdentity = authIdentity ?? authCacheIdentity(pathOptions);
|
|
1626
|
+
const hash = governorScopeHash(host, resolvedAuthIdentity);
|
|
1627
|
+
if (!hash) return { ok: false, reason: "unknown-host" };
|
|
1628
|
+
const currentIdentity = identityProvider ?? (authIdentity === undefined
|
|
1629
|
+
? () => ({ effectiveHost: host, authIdentity: authCacheIdentity(pathOptions) })
|
|
1630
|
+
: null);
|
|
1631
|
+
return {
|
|
1632
|
+
ok: true,
|
|
1633
|
+
value: {
|
|
1634
|
+
hash,
|
|
1635
|
+
path: governorPath(hash, pathOptions),
|
|
1636
|
+
host,
|
|
1637
|
+
authIdentity: resolvedAuthIdentity,
|
|
1638
|
+
identityProvider: currentIdentity,
|
|
1639
|
+
now,
|
|
1640
|
+
kill,
|
|
1641
|
+
},
|
|
1642
|
+
};
|
|
1643
|
+
}
|
|
1644
|
+
|
|
1645
|
+
function scopeNow(scope, nowMs) {
|
|
1646
|
+
return Number.isFinite(nowMs) ? nowMs : Number(scope?.now?.());
|
|
1647
|
+
}
|
|
1648
|
+
|
|
1649
|
+
function governorEffectiveTime(scope, requestedAt) {
|
|
1650
|
+
const currentAt = scopeNow(scope);
|
|
1651
|
+
return Number.isFinite(currentAt) && currentAt >= 0
|
|
1652
|
+
? Math.max(requestedAt, currentAt)
|
|
1653
|
+
: requestedAt;
|
|
1654
|
+
}
|
|
1655
|
+
|
|
1656
|
+
function currentGovernorScope(scope) {
|
|
1657
|
+
if (!scope || typeof scope.path !== "string" || !scope.hash || !normalizeHost(scope.host)) {
|
|
1658
|
+
return { ok: false, reason: "unknown-host" };
|
|
1659
|
+
}
|
|
1660
|
+
if (typeof scope.identityProvider !== "function") return { ok: true, value: scope };
|
|
1661
|
+
let current;
|
|
1662
|
+
try {
|
|
1663
|
+
current = scope.identityProvider();
|
|
1664
|
+
} catch {
|
|
1665
|
+
return { ok: false, reason: "stale" };
|
|
1666
|
+
}
|
|
1667
|
+
const hash = governorScopeHash(current?.effectiveHost, current?.authIdentity);
|
|
1668
|
+
return hash === scope.hash ? { ok: true, value: scope } : { ok: false, reason: "stale" };
|
|
1669
|
+
}
|
|
1670
|
+
|
|
1671
|
+
function emptyGovernorState() {
|
|
1672
|
+
return {
|
|
1673
|
+
version: GOVERNOR_STATE_VERSION,
|
|
1674
|
+
epochs: { core: null, graphql: null },
|
|
1675
|
+
budgets: {},
|
|
1676
|
+
probeClaim: null,
|
|
1677
|
+
probeOutcome: { status: "idle", at: 0, nextAt: 0 },
|
|
1678
|
+
leases: {},
|
|
1679
|
+
intents: {},
|
|
1680
|
+
reservations: {},
|
|
1681
|
+
manualProbe: null,
|
|
1682
|
+
};
|
|
1683
|
+
}
|
|
1684
|
+
|
|
1685
|
+
function finiteTimestamp(value, nowMs, { nullable = false, future = GOVERNOR_MAX_FUTURE_MS } = {}) {
|
|
1686
|
+
if (nullable && value === null) return null;
|
|
1687
|
+
return Number.isFinite(value) && value >= 0 && value <= nowMs + future ? value : undefined;
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
function exactKeys(value, keys) {
|
|
1691
|
+
return isRecord(value) &&
|
|
1692
|
+
Object.keys(value).length === keys.length &&
|
|
1693
|
+
keys.every((key) => Object.hasOwn(value, key));
|
|
1694
|
+
}
|
|
1695
|
+
|
|
1696
|
+
function normalizeGovernorBudget(raw, nowMs) {
|
|
1697
|
+
if (!exactKeys(raw, [
|
|
1698
|
+
"limit", "remaining", "used", "resetMs", "observedAt", "blockUntil",
|
|
1699
|
+
"blockReason", "laneNextAt", "roundRobinCursor", "lastExternalFactor", "epoch",
|
|
1700
|
+
])) return null;
|
|
1701
|
+
const normalized = normalizeBudgetResource(raw);
|
|
1702
|
+
if (
|
|
1703
|
+
!normalized ||
|
|
1704
|
+
normalized.observedAt > nowMs ||
|
|
1705
|
+
finiteTimestamp(normalized.resetMs, nowMs) === undefined ||
|
|
1706
|
+
finiteTimestamp(raw.blockUntil, nowMs, { nullable: true }) === undefined ||
|
|
1707
|
+
finiteTimestamp(raw.laneNextAt, nowMs) === undefined ||
|
|
1708
|
+
(raw.blockReason !== null && !GOVERNOR_BLOCK_REASONS.has(raw.blockReason)) ||
|
|
1709
|
+
(raw.roundRobinCursor !== null && !validGovernorId(raw.roundRobinCursor)) ||
|
|
1710
|
+
!Number.isFinite(raw.lastExternalFactor) || raw.lastExternalFactor < 1 ||
|
|
1711
|
+
!validGovernorEpoch(raw.epoch)
|
|
1712
|
+
) return null;
|
|
1713
|
+
return {
|
|
1714
|
+
...normalized,
|
|
1715
|
+
blockUntil: raw.blockUntil,
|
|
1716
|
+
blockReason: raw.blockReason,
|
|
1717
|
+
laneNextAt: raw.laneNextAt,
|
|
1718
|
+
roundRobinCursor: raw.roundRobinCursor,
|
|
1719
|
+
lastExternalFactor: raw.lastExternalFactor,
|
|
1720
|
+
epoch: raw.epoch,
|
|
1721
|
+
};
|
|
1722
|
+
}
|
|
1723
|
+
|
|
1724
|
+
function normalizeGovernorLease(raw, nowMs) {
|
|
1725
|
+
if (!exactKeys(raw, ["expiresAt", "floorMs", "activeTab", "phaseSeed", "demand"])) return null;
|
|
1726
|
+
const phaseSeed = normalizePhaseSeed(raw.phaseSeed);
|
|
1727
|
+
const demand = exactResourceCosts(raw.demand);
|
|
1728
|
+
if (
|
|
1729
|
+
finiteTimestamp(raw.expiresAt, nowMs) === undefined ||
|
|
1730
|
+
!Number.isFinite(raw.floorMs) || raw.floorMs < MIN_REFRESH_SECONDS * 1000 ||
|
|
1731
|
+
!TAB_KEYS.includes(raw.activeTab) ||
|
|
1732
|
+
!phaseSeed || !validGovernorId(phaseSeed.seed) || phaseSeed.registeredAt > nowMs ||
|
|
1733
|
+
!demand
|
|
1734
|
+
) return null;
|
|
1735
|
+
return { expiresAt: raw.expiresAt, floorMs: raw.floorMs, activeTab: raw.activeTab, phaseSeed, demand };
|
|
1736
|
+
}
|
|
1737
|
+
|
|
1738
|
+
function normalizeGovernorIntent(raw, nowMs) {
|
|
1739
|
+
if (!exactKeys(raw, ["leaseId", "tab", "priority", "costs", "requestedAt", "expiresAt"])) return null;
|
|
1740
|
+
const expectedCosts = TAB_KEYS.includes(raw.tab)
|
|
1741
|
+
? tabRequestCost(raw.tab)
|
|
1742
|
+
: operationCost(raw.tab);
|
|
1743
|
+
const suppliedCosts = exactResourceCosts(raw.costs);
|
|
1744
|
+
const costs = expectedCosts && suppliedCosts &&
|
|
1745
|
+
RATE_RESOURCES.every((resource) => expectedCosts[resource] === suppliedCosts[resource])
|
|
1746
|
+
? expectedCosts
|
|
1747
|
+
: null;
|
|
1748
|
+
if (
|
|
1749
|
+
!validGovernorId(raw.leaseId) ||
|
|
1750
|
+
typeof raw.tab !== "string" || intentPriority(raw) === null || !costs ||
|
|
1751
|
+
finiteTimestamp(raw.requestedAt, nowMs) === undefined || raw.requestedAt > nowMs ||
|
|
1752
|
+
finiteTimestamp(raw.expiresAt, nowMs) === undefined
|
|
1753
|
+
) return null;
|
|
1754
|
+
return { ...raw, costs };
|
|
1755
|
+
}
|
|
1756
|
+
|
|
1757
|
+
function normalizeGovernorReservation(raw, nowMs) {
|
|
1758
|
+
if (!exactKeys(raw, [
|
|
1759
|
+
"leaseId", "intentId", "costs", "actualCosts", "notBefore", "status", "epochs",
|
|
1760
|
+
"startedAt", "completedAt", "outcome",
|
|
1761
|
+
])) return null;
|
|
1762
|
+
const costs = exactResourceCosts(raw.costs);
|
|
1763
|
+
const actualCosts = raw.actualCosts === null ? null : exactResourceCosts(raw.actualCosts);
|
|
1764
|
+
const epochs = exactKeys(raw.epochs, RATE_RESOURCES) &&
|
|
1765
|
+
RATE_RESOURCES.every((resource) => raw.epochs[resource] === null || validGovernorEpoch(raw.epochs[resource]))
|
|
1766
|
+
? { ...raw.epochs }
|
|
1767
|
+
: null;
|
|
1768
|
+
if (
|
|
1769
|
+
!validGovernorId(raw.leaseId) || !validGovernorId(raw.intentId) ||
|
|
1770
|
+
!costs || actualCosts === null && raw.actualCosts !== null || !epochs ||
|
|
1771
|
+
finiteTimestamp(raw.notBefore, nowMs) === undefined ||
|
|
1772
|
+
!GOVERNOR_RESERVATION_STATUSES.has(raw.status) ||
|
|
1773
|
+
finiteTimestamp(raw.startedAt, nowMs, { nullable: true }) === undefined ||
|
|
1774
|
+
finiteTimestamp(raw.completedAt, nowMs, { nullable: true }) === undefined ||
|
|
1775
|
+
(raw.startedAt !== null && raw.startedAt > nowMs) ||
|
|
1776
|
+
(raw.completedAt !== null && raw.completedAt > nowMs) ||
|
|
1777
|
+
(raw.outcome !== null && !GOVERNOR_OUTCOMES.has(raw.outcome))
|
|
1778
|
+
) return null;
|
|
1779
|
+
if (actualCosts && RATE_RESOURCES.some((resource) => actualCosts[resource] > costs[resource])) return null;
|
|
1780
|
+
return { ...raw, costs, actualCosts, epochs };
|
|
1781
|
+
}
|
|
1782
|
+
|
|
1783
|
+
function normalizeProbeClaim(raw, nowMs) {
|
|
1784
|
+
if (raw === null) return null;
|
|
1785
|
+
if (!exactKeys(raw, ["ownerLeaseId", "nonce", "leaseUntil", "nextAt", "claimAt", "startedReservationIds"])) {
|
|
1786
|
+
return undefined;
|
|
1787
|
+
}
|
|
1788
|
+
if (
|
|
1789
|
+
!validGovernorId(raw.ownerLeaseId) || !validGovernorId(raw.nonce) ||
|
|
1790
|
+
finiteTimestamp(raw.leaseUntil, nowMs) === undefined ||
|
|
1791
|
+
finiteTimestamp(raw.nextAt, nowMs) === undefined ||
|
|
1792
|
+
finiteTimestamp(raw.claimAt, nowMs) === undefined || raw.claimAt > nowMs ||
|
|
1793
|
+
!Array.isArray(raw.startedReservationIds) || raw.startedReservationIds.length > GOVERNOR_MAX_RESERVATIONS ||
|
|
1794
|
+
raw.startedReservationIds.some((id) => !id.startsWith("reservation:") || !validGovernorId(id.slice(12)))
|
|
1795
|
+
) return undefined;
|
|
1796
|
+
return { ...raw, startedReservationIds: [...new Set(raw.startedReservationIds)] };
|
|
1797
|
+
}
|
|
1798
|
+
|
|
1799
|
+
function normalizeManualProbe(raw, nowMs) {
|
|
1800
|
+
if (raw === null) return null;
|
|
1801
|
+
if (!exactKeys(raw, ["requestedEpoch", "baselineObservedAt", "satisfiedAt"])) return undefined;
|
|
1802
|
+
if (
|
|
1803
|
+
!validGovernorEpoch(raw.requestedEpoch) ||
|
|
1804
|
+
finiteTimestamp(raw.baselineObservedAt, nowMs) === undefined || raw.baselineObservedAt > nowMs ||
|
|
1805
|
+
finiteTimestamp(raw.satisfiedAt, nowMs, { nullable: true }) === undefined ||
|
|
1806
|
+
(raw.satisfiedAt !== null && raw.satisfiedAt > nowMs)
|
|
1807
|
+
) return undefined;
|
|
1808
|
+
return { ...raw };
|
|
1809
|
+
}
|
|
1810
|
+
|
|
1811
|
+
function normalizeGovernorState(raw, nowMs, { prune = true } = {}) {
|
|
1812
|
+
if (!exactKeys(raw, [
|
|
1813
|
+
"version", "epochs", "budgets", "probeClaim", "probeOutcome", "leases",
|
|
1814
|
+
"intents", "reservations", "manualProbe",
|
|
1815
|
+
]) || raw.version !== GOVERNOR_STATE_VERSION) return null;
|
|
1816
|
+
if (
|
|
1817
|
+
!exactKeys(raw.epochs, RATE_RESOURCES) ||
|
|
1818
|
+
RATE_RESOURCES.some((resource) => raw.epochs[resource] !== null && !validGovernorEpoch(raw.epochs[resource])) ||
|
|
1819
|
+
!isRecord(raw.budgets) || Object.keys(raw.budgets).some((resource) => !RATE_RESOURCES.includes(resource)) ||
|
|
1820
|
+
!isRecord(raw.leases) || !isRecord(raw.intents) || !isRecord(raw.reservations) ||
|
|
1821
|
+
!exactKeys(raw.probeOutcome, ["status", "at", "nextAt"]) ||
|
|
1822
|
+
!GOVERNOR_PROBE_STATUSES.has(raw.probeOutcome.status) ||
|
|
1823
|
+
finiteTimestamp(raw.probeOutcome.at, nowMs) === undefined ||
|
|
1824
|
+
raw.probeOutcome.at > nowMs ||
|
|
1825
|
+
finiteTimestamp(raw.probeOutcome.nextAt, nowMs) === undefined
|
|
1826
|
+
) return null;
|
|
1827
|
+
|
|
1828
|
+
const state = emptyGovernorState();
|
|
1829
|
+
state.epochs = { ...raw.epochs };
|
|
1830
|
+
state.probeOutcome = { ...raw.probeOutcome };
|
|
1831
|
+
for (const resource of Object.keys(raw.budgets)) {
|
|
1832
|
+
const budget = normalizeGovernorBudget(raw.budgets[resource], nowMs);
|
|
1833
|
+
if (!budget) return null;
|
|
1834
|
+
state.budgets[resource] = budget;
|
|
1835
|
+
}
|
|
1836
|
+
for (const [id, rawLease] of Object.entries(raw.leases)) {
|
|
1837
|
+
if (!validGovernorId(id)) return null;
|
|
1838
|
+
const lease = normalizeGovernorLease(rawLease, nowMs);
|
|
1839
|
+
if (!lease || lease.phaseSeed.seed !== id) return null;
|
|
1840
|
+
if (!prune || lease.expiresAt > nowMs) state.leases[id] = lease;
|
|
1841
|
+
}
|
|
1842
|
+
if (Object.keys(state.leases).length > GOVERNOR_MAX_LEASES) return null;
|
|
1843
|
+
for (const [id, rawIntent] of Object.entries(raw.intents)) {
|
|
1844
|
+
if (!validGovernorId(id)) return null;
|
|
1845
|
+
const intent = normalizeGovernorIntent(rawIntent, nowMs);
|
|
1846
|
+
if (!intent) return null;
|
|
1847
|
+
if (!prune || (intent.expiresAt > nowMs && state.leases[intent.leaseId])) state.intents[id] = intent;
|
|
1848
|
+
}
|
|
1849
|
+
if (Object.keys(state.intents).length > GOVERNOR_MAX_INTENTS) return null;
|
|
1850
|
+
for (const [id, rawReservation] of Object.entries(raw.reservations)) {
|
|
1851
|
+
if (!id.startsWith("reservation:") || !validGovernorId(id.slice(12))) return null;
|
|
1852
|
+
const reservation = normalizeGovernorReservation(rawReservation, nowMs);
|
|
1853
|
+
if (!reservation || id !== `reservation:${reservation.intentId}`) return null;
|
|
1854
|
+
if (
|
|
1855
|
+
!prune || reservation.status === "started" || reservation.status === "completed" ||
|
|
1856
|
+
(reservation.status === "scheduled" && state.leases[reservation.leaseId])
|
|
1857
|
+
) state.reservations[id] = reservation;
|
|
1858
|
+
}
|
|
1859
|
+
if (Object.keys(state.reservations).length > GOVERNOR_MAX_RESERVATIONS) return null;
|
|
1860
|
+
state.probeClaim = normalizeProbeClaim(raw.probeClaim, nowMs);
|
|
1861
|
+
if (state.probeClaim === undefined) return null;
|
|
1862
|
+
if (state.probeClaim?.leaseUntil <= nowMs) state.probeClaim = null;
|
|
1863
|
+
state.manualProbe = normalizeManualProbe(raw.manualProbe, nowMs);
|
|
1864
|
+
if (state.manualProbe === undefined) return null;
|
|
1865
|
+
return state;
|
|
1866
|
+
}
|
|
1867
|
+
|
|
1868
|
+
function serializeGovernorState(state) {
|
|
1869
|
+
return `${JSON.stringify(state)}\n`;
|
|
1870
|
+
}
|
|
1871
|
+
|
|
1872
|
+
function readGovernorState(path, nowMs) {
|
|
1873
|
+
let raw;
|
|
1874
|
+
try {
|
|
1875
|
+
raw = readFileSync(path, "utf8");
|
|
1876
|
+
} catch (error) {
|
|
1877
|
+
return error?.code === "ENOENT"
|
|
1878
|
+
? { ok: true, value: emptyGovernorState(), missing: true }
|
|
1879
|
+
: { ok: false, reason: "unwritable" };
|
|
1880
|
+
}
|
|
1881
|
+
try {
|
|
1882
|
+
const value = normalizeGovernorState(JSON.parse(raw), nowMs);
|
|
1883
|
+
return value ? { ok: true, value } : { ok: false, reason: "corrupt" };
|
|
1884
|
+
} catch {
|
|
1885
|
+
return { ok: false, reason: "corrupt" };
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
|
|
1889
|
+
function writeGovernorState(path, state) {
|
|
1890
|
+
const parent = dirname(path);
|
|
1891
|
+
let tempPath = null;
|
|
1892
|
+
try {
|
|
1893
|
+
mkdirSync(parent, { recursive: true, mode: 0o700 });
|
|
1894
|
+
chmodSync(parent, 0o700);
|
|
1895
|
+
tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`;
|
|
1896
|
+
writeFileSync(tempPath, serializeGovernorState(state), { encoding: "utf8", mode: 0o600, flag: "wx" });
|
|
1897
|
+
chmodSync(tempPath, 0o600);
|
|
1898
|
+
renameSync(tempPath, path);
|
|
1899
|
+
chmodSync(path, 0o600);
|
|
1900
|
+
return { ok: true };
|
|
1901
|
+
} catch {
|
|
1902
|
+
if (tempPath !== null) {
|
|
1903
|
+
try { unlinkSync(tempPath); } catch { /* exact private temporary file only */ }
|
|
1904
|
+
}
|
|
1905
|
+
return { ok: false, reason: "unwritable" };
|
|
1906
|
+
}
|
|
1907
|
+
}
|
|
1908
|
+
|
|
1909
|
+
function lockOwner(path) {
|
|
1910
|
+
try {
|
|
1911
|
+
const owner = JSON.parse(readFileSync(path, "utf8"));
|
|
1912
|
+
return exactKeys(owner, ["pid", "nonce"]) && Number.isSafeInteger(owner.pid) && owner.pid > 0 &&
|
|
1913
|
+
typeof owner.nonce === "string" && owner.nonce.length > 0 ? owner : null;
|
|
1914
|
+
} catch {
|
|
1915
|
+
return null;
|
|
1916
|
+
}
|
|
1917
|
+
}
|
|
1918
|
+
|
|
1919
|
+
function pidIsDead(pid, kill = process.kill.bind(process)) {
|
|
1920
|
+
try {
|
|
1921
|
+
kill(pid, 0);
|
|
1922
|
+
return false;
|
|
1923
|
+
} catch (error) {
|
|
1924
|
+
if (error?.code === "ESRCH") return true;
|
|
1925
|
+
return false;
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
|
|
1929
|
+
function releaseGovernorLock(lockPath, nonce) {
|
|
1930
|
+
const owner = lockOwner(lockPath);
|
|
1931
|
+
if (!owner || owner.nonce !== nonce) return false;
|
|
1932
|
+
const releasePath = `${lockPath}.release-${nonce}`;
|
|
1933
|
+
try {
|
|
1934
|
+
renameSync(lockPath, releasePath);
|
|
1935
|
+
unlinkSync(releasePath);
|
|
1936
|
+
return true;
|
|
1937
|
+
} catch {
|
|
1938
|
+
return false;
|
|
1939
|
+
}
|
|
1940
|
+
}
|
|
1941
|
+
|
|
1942
|
+
function governorRecoveryPaths(lockPath) {
|
|
1943
|
+
try {
|
|
1944
|
+
const prefix = `${basename(lockPath)}.recovery-`;
|
|
1945
|
+
return readdirSync(dirname(lockPath))
|
|
1946
|
+
.filter((name) => name.startsWith(prefix))
|
|
1947
|
+
.map((name) => join(dirname(lockPath), name));
|
|
1948
|
+
} catch {
|
|
1949
|
+
return [];
|
|
1950
|
+
}
|
|
1951
|
+
}
|
|
1952
|
+
|
|
1953
|
+
function governorRecoveryActive(lockPath, kill) {
|
|
1954
|
+
let active = false;
|
|
1955
|
+
for (const recoveryPath of governorRecoveryPaths(lockPath)) {
|
|
1956
|
+
const owner = lockOwner(recoveryPath);
|
|
1957
|
+
if (!owner || !pidIsDead(owner.pid, kill)) {
|
|
1958
|
+
active = true;
|
|
1959
|
+
continue;
|
|
1960
|
+
}
|
|
1961
|
+
releaseGovernorLock(recoveryPath, owner.nonce);
|
|
1962
|
+
}
|
|
1963
|
+
return active || governorRecoveryPaths(lockPath).length > 0;
|
|
1964
|
+
}
|
|
1965
|
+
|
|
1966
|
+
function sameLockOwner(left, right) {
|
|
1967
|
+
return Boolean(left) && Boolean(right) && left.pid === right.pid && left.nonce === right.nonce;
|
|
1968
|
+
}
|
|
1969
|
+
|
|
1970
|
+
function observeGovernorArtifact(observer, kind, path) {
|
|
1971
|
+
try { observer?.(kind, path); } catch { /* test observation cannot affect locking */ }
|
|
1972
|
+
}
|
|
1973
|
+
|
|
1974
|
+
function quarantineDeadGovernorLock(lockPath, expectedOwner, {
|
|
1975
|
+
pid = process.pid,
|
|
1976
|
+
kill = process.kill.bind(process),
|
|
1977
|
+
observeArtifact = null,
|
|
1978
|
+
} = {}) {
|
|
1979
|
+
const recoveryNonce = randomUUID();
|
|
1980
|
+
const recoveryPath = `${lockPath}.recovery-${recoveryNonce}`;
|
|
1981
|
+
let descriptor;
|
|
1982
|
+
try {
|
|
1983
|
+
descriptor = openSync(recoveryPath, "wx", 0o600);
|
|
1984
|
+
try {
|
|
1985
|
+
writeFileSync(descriptor, JSON.stringify({ pid, nonce: recoveryNonce }), "utf8");
|
|
1986
|
+
} finally {
|
|
1987
|
+
closeSync(descriptor);
|
|
1988
|
+
}
|
|
1989
|
+
chmodSync(recoveryPath, 0o600);
|
|
1990
|
+
observeGovernorArtifact(observeArtifact, "recovery", recoveryPath);
|
|
1991
|
+
} catch (error) {
|
|
1992
|
+
if (descriptor !== undefined) {
|
|
1993
|
+
try { closeSync(descriptor); } catch { /* descriptor was already closed */ }
|
|
1994
|
+
}
|
|
1995
|
+
return error?.code === "EEXIST" ? "busy" : "failed";
|
|
1996
|
+
}
|
|
1997
|
+
|
|
1998
|
+
const quarantinePath = `${lockPath}.quarantine-${randomUUID()}`;
|
|
1999
|
+
try {
|
|
2000
|
+
// Unique recovery markers make every new owner recheck after acquisition.
|
|
2001
|
+
// A killed recovery process leaves a uniquely named marker that another
|
|
2002
|
+
// process can remove only after its PID is confirmed dead; the path can
|
|
2003
|
+
// never be reused by a successor.
|
|
2004
|
+
// Re-read both fields after taking that marker and again immediately before
|
|
2005
|
+
// rename, so an owner that changed since the initial ESRCH result is never
|
|
2006
|
+
// selected as the abandoned lock.
|
|
2007
|
+
const confirmed = lockOwner(lockPath);
|
|
2008
|
+
if (!sameLockOwner(confirmed, expectedOwner) || !pidIsDead(confirmed.pid, kill)) return "changed";
|
|
2009
|
+
const beforeRename = lockOwner(lockPath);
|
|
2010
|
+
if (!sameLockOwner(beforeRename, expectedOwner)) return "changed";
|
|
2011
|
+
renameSync(lockPath, quarantinePath);
|
|
2012
|
+
observeGovernorArtifact(observeArtifact, "quarantine", quarantinePath);
|
|
2013
|
+
const quarantined = lockOwner(quarantinePath);
|
|
2014
|
+
if (!sameLockOwner(quarantined, expectedOwner)) {
|
|
2015
|
+
// Acquisitions that raced the recovery marker cannot enter their critical
|
|
2016
|
+
// section. Restoring here preserves that successor rather than deleting
|
|
2017
|
+
// or leaving it under an abandoned quarantine name.
|
|
2018
|
+
renameSync(quarantinePath, lockPath);
|
|
2019
|
+
return "changed";
|
|
2020
|
+
}
|
|
2021
|
+
unlinkSync(quarantinePath);
|
|
2022
|
+
return "quarantined";
|
|
2023
|
+
} catch (error) {
|
|
2024
|
+
return error?.code === "ENOENT" ? "changed" : "failed";
|
|
2025
|
+
} finally {
|
|
2026
|
+
releaseGovernorLock(recoveryPath, recoveryNonce);
|
|
2027
|
+
}
|
|
2028
|
+
}
|
|
2029
|
+
|
|
2030
|
+
function withGovernorLock(scope, operation, {
|
|
2031
|
+
pid = process.pid,
|
|
2032
|
+
nonce = randomUUID(),
|
|
2033
|
+
waitMs = GOVERNOR_LOCK_WAIT_MS,
|
|
2034
|
+
kill = scope?.kill ?? process.kill.bind(process),
|
|
2035
|
+
observeArtifact = null,
|
|
2036
|
+
} = {}) {
|
|
2037
|
+
const current = currentGovernorScope(scope);
|
|
2038
|
+
if (!current.ok) return current;
|
|
2039
|
+
const lockPath = `${scope.path}.lock`;
|
|
2040
|
+
const deadline = Date.now() + waitMs;
|
|
2041
|
+
try {
|
|
2042
|
+
mkdirSync(dirname(scope.path), { recursive: true, mode: 0o700 });
|
|
2043
|
+
chmodSync(dirname(scope.path), 0o700);
|
|
2044
|
+
} catch {
|
|
2045
|
+
return { ok: false, reason: "unwritable" };
|
|
2046
|
+
}
|
|
2047
|
+
const waitForLock = () => {
|
|
2048
|
+
if (Date.now() >= deadline) return false;
|
|
2049
|
+
Atomics.wait(persistenceWaitCell, 0, 0, 5);
|
|
2050
|
+
return true;
|
|
2051
|
+
};
|
|
2052
|
+
for (;;) {
|
|
2053
|
+
if (governorRecoveryActive(lockPath, kill)) {
|
|
2054
|
+
if (!waitForLock()) return { ok: false, reason: "busy" };
|
|
2055
|
+
continue;
|
|
2056
|
+
}
|
|
2057
|
+
try {
|
|
2058
|
+
const descriptor = openSync(lockPath, "wx", 0o600);
|
|
2059
|
+
try {
|
|
2060
|
+
writeFileSync(descriptor, JSON.stringify({ pid, nonce }), "utf8");
|
|
2061
|
+
} finally {
|
|
2062
|
+
closeSync(descriptor);
|
|
2063
|
+
}
|
|
2064
|
+
chmodSync(lockPath, 0o600);
|
|
2065
|
+
observeGovernorArtifact(observeArtifact, "canonical", lockPath);
|
|
2066
|
+
if (governorRecoveryActive(lockPath, kill)) {
|
|
2067
|
+
releaseGovernorLock(lockPath, nonce);
|
|
2068
|
+
if (!waitForLock()) return { ok: false, reason: "busy" };
|
|
2069
|
+
continue;
|
|
2070
|
+
}
|
|
2071
|
+
try {
|
|
2072
|
+
return operation();
|
|
2073
|
+
} finally {
|
|
2074
|
+
releaseGovernorLock(lockPath, nonce);
|
|
2075
|
+
}
|
|
2076
|
+
} catch (error) {
|
|
2077
|
+
if (error?.code !== "EEXIST") return { ok: false, reason: "unwritable" };
|
|
2078
|
+
const owner = lockOwner(lockPath);
|
|
2079
|
+
// The creator owns the canonical path as soon as open("wx") succeeds,
|
|
2080
|
+
// before its small JSON owner record is fully visible. Treat an unreadable
|
|
2081
|
+
// record as owned during the bounded wait; stealing it could overlap the
|
|
2082
|
+
// creator, while returning busy is always fail-closed.
|
|
2083
|
+
if (!owner) {
|
|
2084
|
+
if (!waitForLock()) return { ok: false, reason: "busy" };
|
|
2085
|
+
continue;
|
|
2086
|
+
}
|
|
2087
|
+
if (pidIsDead(owner.pid, kill)) {
|
|
2088
|
+
const recovery = quarantineDeadGovernorLock(lockPath, owner, { pid, kill, observeArtifact });
|
|
2089
|
+
if (recovery === "quarantined" || recovery === "changed") continue;
|
|
2090
|
+
if (!waitForLock()) return { ok: false, reason: "busy" };
|
|
2091
|
+
continue;
|
|
2092
|
+
}
|
|
2093
|
+
if (!waitForLock()) return { ok: false, reason: "busy" };
|
|
2094
|
+
}
|
|
2095
|
+
}
|
|
2096
|
+
}
|
|
2097
|
+
|
|
2098
|
+
function mutateGovernor(scope, nowMs, mutate) {
|
|
2099
|
+
const requestedAt = scopeNow(scope, nowMs);
|
|
2100
|
+
if (!Number.isFinite(requestedAt) || requestedAt < 0) return { ok: false, reason: "corrupt" };
|
|
2101
|
+
return withGovernorLock(scope, () => {
|
|
2102
|
+
// Another process can persist a lease while this caller waits for the
|
|
2103
|
+
// lock. Re-read the clock after exclusivity is acquired so that valid work
|
|
2104
|
+
// committed during that wait does not look future-dated to normalization.
|
|
2105
|
+
// Explicit future times and injected fake clocks remain authoritative.
|
|
2106
|
+
const at = governorEffectiveTime(scope, requestedAt);
|
|
2107
|
+
// Scope can change while this process waits for the file lock. Admission
|
|
2108
|
+
// must bind to the host/account identity observed while exclusivity is
|
|
2109
|
+
// held, immediately before it reads and mutates the shared state.
|
|
2110
|
+
const current = currentGovernorScope(scope);
|
|
2111
|
+
if (!current.ok) return current;
|
|
2112
|
+
const loaded = readGovernorState(scope.path, at);
|
|
2113
|
+
if (!loaded.ok) return loaded;
|
|
2114
|
+
const state = loaded.value;
|
|
2115
|
+
const value = mutate(state, at);
|
|
2116
|
+
if (value?.ok === false && value.write !== true) return value;
|
|
2117
|
+
const normalized = normalizeGovernorState(state, at);
|
|
2118
|
+
if (!normalized) return { ok: false, reason: "corrupt" };
|
|
2119
|
+
const written = writeGovernorState(scope.path, normalized);
|
|
2120
|
+
if (!written.ok) return written;
|
|
2121
|
+
return value?.ok === false ? { ok: false, reason: value.reason } : { ok: true, value: value?.value };
|
|
2122
|
+
});
|
|
2123
|
+
}
|
|
2124
|
+
|
|
2125
|
+
function scheduleGovernorState(state, nowMs) {
|
|
2126
|
+
if (Object.keys(state.intents).length === 0) return { grants: [], denied: [] };
|
|
2127
|
+
let reservationCount = Object.keys(state.reservations).length;
|
|
2128
|
+
const lanes = Object.fromEntries(RATE_RESOURCES.flatMap((resource) =>
|
|
2129
|
+
state.budgets[resource] ? [[resource, { nextAt: state.budgets[resource].laneNextAt }]] : [],
|
|
2130
|
+
));
|
|
2131
|
+
const cursors = Object.fromEntries(RATE_RESOURCES.flatMap((resource) =>
|
|
2132
|
+
state.budgets[resource]?.roundRobinCursor
|
|
2133
|
+
? [[resource, state.budgets[resource].roundRobinCursor]]
|
|
2134
|
+
: [],
|
|
2135
|
+
));
|
|
2136
|
+
const deferredBackground = state.probeOutcome.status === "failed"
|
|
2137
|
+
? Object.entries(state.intents)
|
|
2138
|
+
.filter(([, intent]) => intentPriority(intent) === REQUEST_PRIORITIES.background)
|
|
2139
|
+
.map(([id]) => id)
|
|
2140
|
+
: [];
|
|
2141
|
+
const deferred = new Set(deferredBackground);
|
|
2142
|
+
const result = scheduleIntents({
|
|
2143
|
+
intents: Object.entries(state.intents)
|
|
2144
|
+
.filter(([id]) => !deferred.has(id))
|
|
2145
|
+
.map(([id, intent]) => ({
|
|
2146
|
+
id,
|
|
2147
|
+
...intent,
|
|
2148
|
+
tab: TAB_KEYS.includes(intent.tab) ? intent.tab : undefined,
|
|
2149
|
+
})),
|
|
2150
|
+
leases: state.leases,
|
|
2151
|
+
budgets: state.budgets,
|
|
2152
|
+
reservations: Object.entries(state.reservations).map(([id, reservation]) => ({ id, ...reservation })),
|
|
2153
|
+
lanes,
|
|
2154
|
+
cursors,
|
|
2155
|
+
nowMs,
|
|
2156
|
+
maxGrants: GOVERNOR_MAX_RESERVATIONS - reservationCount,
|
|
2157
|
+
});
|
|
2158
|
+
result.denied.push(...deferredBackground.map((intentId) => ({
|
|
2159
|
+
intentId,
|
|
2160
|
+
mode: "paused",
|
|
2161
|
+
reason: "probe-failed",
|
|
2162
|
+
})));
|
|
2163
|
+
for (const id of result.prunedIntentIds) delete state.intents[id];
|
|
2164
|
+
for (const grant of result.grants) {
|
|
2165
|
+
if (reservationCount >= GOVERNOR_MAX_RESERVATIONS) break;
|
|
2166
|
+
state.reservations[grant.id] = {
|
|
2167
|
+
leaseId: grant.leaseId,
|
|
2168
|
+
intentId: grant.intentId,
|
|
2169
|
+
costs: grant.costs,
|
|
2170
|
+
actualCosts: null,
|
|
2171
|
+
notBefore: grant.notBefore,
|
|
2172
|
+
status: "scheduled",
|
|
2173
|
+
epochs: { core: grant.epochs.core ?? null, graphql: grant.epochs.graphql ?? null },
|
|
2174
|
+
startedAt: null,
|
|
2175
|
+
completedAt: null,
|
|
2176
|
+
outcome: null,
|
|
2177
|
+
};
|
|
2178
|
+
reservationCount += 1;
|
|
2179
|
+
delete state.intents[grant.intentId];
|
|
2180
|
+
}
|
|
2181
|
+
for (const resource of RATE_RESOURCES) {
|
|
2182
|
+
if (!state.budgets[resource]) continue;
|
|
2183
|
+
state.budgets[resource].laneNextAt = result.lanes[resource]?.nextAt ?? state.budgets[resource].laneNextAt;
|
|
2184
|
+
state.budgets[resource].roundRobinCursor = result.cursors[resource] ?? state.budgets[resource].roundRobinCursor;
|
|
2185
|
+
}
|
|
2186
|
+
return result;
|
|
2187
|
+
}
|
|
2188
|
+
|
|
2189
|
+
function registerLease(scope, lease) {
|
|
2190
|
+
return mutateGovernor(scope, lease?.phaseSeed?.registeredAt, (state, nowMs) => {
|
|
2191
|
+
if (!isRecord(lease) || !validGovernorId(lease.id) || lease.phaseSeed?.seed !== lease.id) {
|
|
2192
|
+
return { ok: false, reason: "corrupt" };
|
|
2193
|
+
}
|
|
2194
|
+
const normalized = normalizeGovernorLease({
|
|
2195
|
+
expiresAt: lease.expiresAt,
|
|
2196
|
+
floorMs: lease.floorMs,
|
|
2197
|
+
activeTab: lease.activeTab,
|
|
2198
|
+
phaseSeed: lease.phaseSeed,
|
|
2199
|
+
demand: lease.demand,
|
|
2200
|
+
}, nowMs);
|
|
2201
|
+
if (!normalized || normalized.expiresAt <= nowMs) return { ok: false, reason: "stale" };
|
|
2202
|
+
if (!state.leases[lease.id] && Object.keys(state.leases).length >= GOVERNOR_MAX_LEASES) {
|
|
2203
|
+
return { ok: false, reason: "busy" };
|
|
2204
|
+
}
|
|
2205
|
+
state.leases[lease.id] = normalized;
|
|
2206
|
+
return { value: { leaseId: lease.id, expiresAt: normalized.expiresAt } };
|
|
2207
|
+
});
|
|
2208
|
+
}
|
|
2209
|
+
|
|
2210
|
+
function heartbeatLease(scope, leaseId, demand, nowMs, activeTab = null) {
|
|
2211
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2212
|
+
const lease = state.leases[leaseId];
|
|
2213
|
+
const costs = exactResourceCosts(demand);
|
|
2214
|
+
if (!lease || !costs || (activeTab !== null && !TAB_KEYS.includes(activeTab))) {
|
|
2215
|
+
return { ok: false, reason: "stale" };
|
|
2216
|
+
}
|
|
2217
|
+
lease.expiresAt = at + GOVERNOR_LEASE_TTL_MS;
|
|
2218
|
+
lease.demand = costs;
|
|
2219
|
+
if (activeTab !== null) lease.activeTab = activeTab;
|
|
2220
|
+
scheduleGovernorState(state, at);
|
|
2221
|
+
return { value: { leaseId, expiresAt: lease.expiresAt, activeTab: lease.activeTab } };
|
|
2222
|
+
});
|
|
2223
|
+
}
|
|
2224
|
+
|
|
2225
|
+
function maintainControlLease(scope, leaseId, floorMs, activeTab, nowMs) {
|
|
2226
|
+
const demand = tabRequestCost(activeTab);
|
|
2227
|
+
if (!demand) return { ok: false, reason: "corrupt" };
|
|
2228
|
+
const renewed = heartbeatLease(scope, leaseId, demand, nowMs, activeTab);
|
|
2229
|
+
if (renewed.ok) return { ...renewed, value: { ...renewed.value, status: "renewed" } };
|
|
2230
|
+
const registered = registerLease(scope, {
|
|
2231
|
+
id: leaseId,
|
|
2232
|
+
expiresAt: nowMs + GOVERNOR_LEASE_TTL_MS,
|
|
2233
|
+
floorMs,
|
|
2234
|
+
activeTab,
|
|
2235
|
+
phaseSeed: { seed: leaseId, registeredAt: nowMs },
|
|
2236
|
+
demand,
|
|
2237
|
+
});
|
|
2238
|
+
return registered.ok
|
|
2239
|
+
? { ...registered, value: { ...registered.value, status: "registered", activeTab } }
|
|
2240
|
+
: registered;
|
|
2241
|
+
}
|
|
2242
|
+
|
|
2243
|
+
function claimProbe(scope, leaseId, nowMs) {
|
|
2244
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2245
|
+
if (!state.leases[leaseId]) return { ok: false, reason: "stale" };
|
|
2246
|
+
if (state.probeClaim && state.probeClaim.leaseUntil > at) {
|
|
2247
|
+
return { value: { status: "waiting", leaseUntil: state.probeClaim.leaseUntil } };
|
|
2248
|
+
}
|
|
2249
|
+
const resetProbeAt = Math.min(...Object.values(state.budgets).map(
|
|
2250
|
+
(budget) => budget.resetMs + BUDGET_RESET_GRACE_MS,
|
|
2251
|
+
));
|
|
2252
|
+
const nextAt = Math.min(state.probeOutcome.nextAt, resetProbeAt);
|
|
2253
|
+
if (nextAt > at) return { value: { status: "waiting", nextAt } };
|
|
2254
|
+
const nonce = randomUUID();
|
|
2255
|
+
const startedReservationIds = Object.entries(state.reservations)
|
|
2256
|
+
.filter(([, reservation]) => reservation.status === "started")
|
|
2257
|
+
.map(([id]) => id);
|
|
2258
|
+
state.probeClaim = {
|
|
2259
|
+
ownerLeaseId: leaseId,
|
|
2260
|
+
nonce,
|
|
2261
|
+
leaseUntil: at + GOVERNOR_PROBE_LEASE_MS,
|
|
2262
|
+
nextAt: at,
|
|
2263
|
+
claimAt: at,
|
|
2264
|
+
startedReservationIds,
|
|
2265
|
+
};
|
|
2266
|
+
state.probeOutcome = { status: "waiting", at, nextAt: at + GOVERNOR_PROBE_LEASE_MS };
|
|
2267
|
+
return { value: { status: "claimed", nonce, leaseUntil: state.probeClaim.leaseUntil, startedReservationIds } };
|
|
2268
|
+
});
|
|
2269
|
+
}
|
|
2270
|
+
|
|
2271
|
+
function renewProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
2272
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2273
|
+
const claim = state.probeClaim;
|
|
2274
|
+
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce || claim.leaseUntil <= at) {
|
|
2275
|
+
return { ok: false, reason: "stale" };
|
|
2276
|
+
}
|
|
2277
|
+
claim.leaseUntil = at + GOVERNOR_ACTIVE_PROBE_LEASE_MS;
|
|
2278
|
+
return { value: { nonce, leaseUntil: claim.leaseUntil } };
|
|
2279
|
+
});
|
|
2280
|
+
}
|
|
2281
|
+
|
|
2282
|
+
function budgetFromProbe(raw, previous, nowMs) {
|
|
2283
|
+
const normalized = normalizeBudgetResource({ ...raw, observedAt: nowMs });
|
|
2284
|
+
if (!normalized || normalized.resetMs <= nowMs) return null;
|
|
2285
|
+
const baseEpoch = `${normalized.limit}:${normalized.resetMs}`;
|
|
2286
|
+
const epoch = previous && previous.resetMs === normalized.resetMs && previous.used <= normalized.used
|
|
2287
|
+
? previous.epoch
|
|
2288
|
+
: previous && previous.resetMs === normalized.resetMs && normalized.used < previous.used
|
|
2289
|
+
? `${baseEpoch}:${nowMs}`
|
|
2290
|
+
: baseEpoch;
|
|
2291
|
+
const epochChanged = !previous || previous.epoch !== epoch;
|
|
2292
|
+
const blockActive = !epochChanged && Number.isFinite(previous.blockUntil) && previous.blockUntil > nowMs;
|
|
2293
|
+
return {
|
|
2294
|
+
...normalized,
|
|
2295
|
+
blockUntil: blockActive ? previous.blockUntil : null,
|
|
2296
|
+
blockReason: blockActive ? previous.blockReason : null,
|
|
2297
|
+
laneNextAt: epochChanged ? nowMs : previous.laneNextAt,
|
|
2298
|
+
roundRobinCursor: previous?.roundRobinCursor ?? null,
|
|
2299
|
+
lastExternalFactor: previous?.lastExternalFactor ?? 1,
|
|
2300
|
+
epoch,
|
|
2301
|
+
};
|
|
2302
|
+
}
|
|
2303
|
+
|
|
2304
|
+
function publishProbe(scope, leaseId, nonce, budgets, nowMs) {
|
|
2305
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2306
|
+
const claim = state.probeClaim;
|
|
2307
|
+
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce || claim.leaseUntil <= at) {
|
|
2308
|
+
return { ok: false, reason: "stale" };
|
|
2309
|
+
}
|
|
2310
|
+
const nextBudgets = {};
|
|
2311
|
+
for (const resource of RATE_RESOURCES) {
|
|
2312
|
+
nextBudgets[resource] = budgetFromProbe(budgets?.[resource], state.budgets[resource], at);
|
|
2313
|
+
if (!nextBudgets[resource]) return { ok: false, reason: "corrupt" };
|
|
2314
|
+
}
|
|
2315
|
+
const nextEpochs = Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
2316
|
+
resource,
|
|
2317
|
+
budgetEpoch(nextBudgets[resource]),
|
|
2318
|
+
]));
|
|
2319
|
+
for (const resource of RATE_RESOURCES) {
|
|
2320
|
+
const previous = state.budgets[resource];
|
|
2321
|
+
if (!previous || nextBudgets[resource].used < previous.used) continue;
|
|
2322
|
+
const sharedCompletedDelta = Object.values(state.reservations)
|
|
2323
|
+
.filter((reservation) => reservation.status === "completed" && reservation.completedAt < claim.claimAt)
|
|
2324
|
+
.reduce((total, reservation) => total + reservationCost(reservation, resource, state.leases, at), 0);
|
|
2325
|
+
const factor = nextExternalFactor({
|
|
2326
|
+
lastExternalFactor: previous.lastExternalFactor,
|
|
2327
|
+
globalUsedDelta: nextBudgets[resource].used - previous.used,
|
|
2328
|
+
sharedCompletedDelta,
|
|
2329
|
+
});
|
|
2330
|
+
if (factor === null) return { ok: false, reason: "corrupt" };
|
|
2331
|
+
nextBudgets[resource].lastExternalFactor = factor;
|
|
2332
|
+
}
|
|
2333
|
+
const resetChanged = RATE_RESOURCES.some((resource) =>
|
|
2334
|
+
state.epochs[resource] !== null && state.epochs[resource] !== nextEpochs[resource],
|
|
2335
|
+
);
|
|
2336
|
+
for (const [id, reservation] of Object.entries(state.reservations)) {
|
|
2337
|
+
if (reservation.status === "completed" && reservation.completedAt < claim.claimAt) {
|
|
2338
|
+
delete state.reservations[id];
|
|
2339
|
+
}
|
|
2340
|
+
}
|
|
2341
|
+
state.budgets = nextBudgets;
|
|
2342
|
+
state.epochs = nextEpochs;
|
|
2343
|
+
state.probeClaim = null;
|
|
2344
|
+
state.probeOutcome = { status: "healthy", at, nextAt: at + BUDGET_PROBE_MS };
|
|
2345
|
+
if (resetChanged) state.manualProbe = null;
|
|
2346
|
+
else if (state.manualProbe &&
|
|
2347
|
+
Object.values(nextEpochs).includes(state.manualProbe.requestedEpoch) &&
|
|
2348
|
+
RATE_RESOURCES.some((resource) => availableForGrant({ budget: nextBudgets[resource], nowMs: at }).spendable === 0)) {
|
|
2349
|
+
state.manualProbe.satisfiedAt = at;
|
|
2350
|
+
}
|
|
2351
|
+
scheduleGovernorState(state, at);
|
|
2352
|
+
return { value: { epochs: nextEpochs, retiredThrough: claim.claimAt } };
|
|
2353
|
+
});
|
|
2354
|
+
}
|
|
1012
2355
|
|
|
1013
|
-
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
2356
|
+
function failProbeClaim(scope, leaseId, nonce, nowMs) {
|
|
2357
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2358
|
+
const claim = state.probeClaim;
|
|
2359
|
+
if (!claim || claim.ownerLeaseId !== leaseId || claim.nonce !== nonce) {
|
|
2360
|
+
return { ok: false, reason: "stale" };
|
|
2361
|
+
}
|
|
2362
|
+
state.probeClaim = null;
|
|
2363
|
+
state.probeOutcome = { status: "failed", at, nextAt: at + BUDGET_PROBE_MS };
|
|
2364
|
+
return { value: { retryAt: state.probeOutcome.nextAt } };
|
|
2365
|
+
});
|
|
2366
|
+
}
|
|
1017
2367
|
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
2368
|
+
function requestManualProbe(scope, leaseId, epoch, observedAt, nowMs) {
|
|
2369
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2370
|
+
if (!state.leases[leaseId] || typeof epoch !== "string" || !Number.isFinite(observedAt)) {
|
|
2371
|
+
return { ok: false, reason: "stale" };
|
|
2372
|
+
}
|
|
2373
|
+
if (state.manualProbe?.requestedEpoch === epoch && state.manualProbe.baselineObservedAt === observedAt) {
|
|
2374
|
+
return { value: { status: state.manualProbe.satisfiedAt === null ? "pending" : "satisfied", ...state.manualProbe } };
|
|
2375
|
+
}
|
|
2376
|
+
state.manualProbe = { requestedEpoch: epoch, baselineObservedAt: observedAt, satisfiedAt: null };
|
|
2377
|
+
state.probeOutcome.nextAt = Math.min(state.probeOutcome.nextAt || at, at);
|
|
2378
|
+
return { value: { status: "pending", ...state.manualProbe } };
|
|
2379
|
+
});
|
|
2380
|
+
}
|
|
1022
2381
|
|
|
1023
|
-
|
|
1024
|
-
|
|
1025
|
-
|
|
1026
|
-
|
|
1027
|
-
|
|
2382
|
+
function registerIntent(scope, intent) {
|
|
2383
|
+
return mutateGovernor(scope, intent?.requestedAt, (state, nowMs) => {
|
|
2384
|
+
if (!isRecord(intent) || !validGovernorId(intent.id)) {
|
|
2385
|
+
return { ok: false, reason: "corrupt" };
|
|
2386
|
+
}
|
|
2387
|
+
const normalized = normalizeGovernorIntent({
|
|
2388
|
+
leaseId: intent.leaseId,
|
|
2389
|
+
tab: intent.tab,
|
|
2390
|
+
priority: intent.priority,
|
|
2391
|
+
costs: intent.costs ?? tabRequestCost(intent.tab),
|
|
2392
|
+
requestedAt: intent.requestedAt,
|
|
2393
|
+
expiresAt: intent.expiresAt,
|
|
2394
|
+
}, nowMs);
|
|
2395
|
+
if (!normalized) return { ok: false, reason: "corrupt" };
|
|
2396
|
+
if (normalized.expiresAt <= nowMs || !state.leases[normalized.leaseId]) {
|
|
2397
|
+
return { ok: false, reason: "stale" };
|
|
2398
|
+
}
|
|
2399
|
+
const duplicate = Object.entries(state.intents).find(([, pending]) =>
|
|
2400
|
+
pending.leaseId === normalized.leaseId && pending.tab === normalized.tab &&
|
|
2401
|
+
pending.priority === normalized.priority,
|
|
2402
|
+
);
|
|
2403
|
+
if (duplicate) return { value: { status: "pending", intentId: duplicate[0], coalesced: true } };
|
|
2404
|
+
const reservationId = `reservation:${intent.id}`;
|
|
2405
|
+
const existingReservation = state.reservations[reservationId];
|
|
2406
|
+
if (existingReservation) return { value: { status: existingReservation.status, reservationId } };
|
|
2407
|
+
if (!state.intents[intent.id] && Object.keys(state.intents).length >= GOVERNOR_MAX_INTENTS) {
|
|
2408
|
+
return { ok: false, reason: "busy" };
|
|
2409
|
+
}
|
|
2410
|
+
state.intents[intent.id] = normalized;
|
|
2411
|
+
const scheduled = scheduleGovernorState(state, nowMs);
|
|
2412
|
+
const reservation = state.reservations[reservationId];
|
|
2413
|
+
const denial = scheduled.denied.find((item) => item.intentId === intent.id);
|
|
2414
|
+
return { value: reservation
|
|
2415
|
+
? { status: "scheduled", reservationId, ...reservation }
|
|
2416
|
+
: {
|
|
2417
|
+
status: denial?.mode ?? "pending",
|
|
2418
|
+
intentId: intent.id,
|
|
2419
|
+
resource: denial?.resource ?? null,
|
|
2420
|
+
reason: denial?.reason ?? "budget-unknown",
|
|
2421
|
+
resetMs: denial?.resetMs ?? null,
|
|
2422
|
+
retryAt: denial?.retryAt ?? denial?.notBefore ?? null,
|
|
2423
|
+
notBefore: denial?.notBefore ?? null,
|
|
2424
|
+
} };
|
|
2425
|
+
});
|
|
2426
|
+
}
|
|
1028
2427
|
|
|
1029
|
-
|
|
1030
|
-
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
const
|
|
2428
|
+
function readIntentDecision(scope, intentId, nowMs) {
|
|
2429
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2430
|
+
const scheduled = scheduleGovernorState(state, at);
|
|
2431
|
+
const reservationId = `reservation:${intentId}`;
|
|
2432
|
+
const reservation = state.reservations[reservationId];
|
|
2433
|
+
if (reservation) return { value: { status: reservation.status, reservationId, ...reservation } };
|
|
2434
|
+
if (!state.intents[intentId]) return { ok: false, reason: "stale" };
|
|
2435
|
+
const denial = scheduled.denied.find((item) => item.intentId === intentId);
|
|
2436
|
+
return { value: {
|
|
2437
|
+
status: denial?.mode ?? "pending",
|
|
2438
|
+
resource: denial?.resource ?? null,
|
|
2439
|
+
reason: denial?.reason ?? "budget-unknown",
|
|
2440
|
+
resetMs: denial?.resetMs ?? null,
|
|
2441
|
+
retryAt: denial?.retryAt ?? denial?.notBefore ?? null,
|
|
2442
|
+
notBefore: denial?.notBefore ?? null,
|
|
2443
|
+
} };
|
|
2444
|
+
});
|
|
2445
|
+
}
|
|
1034
2446
|
|
|
1035
|
-
|
|
1036
|
-
|
|
1037
|
-
|
|
1038
|
-
|
|
2447
|
+
function cancelIntent(scope, intentId, nowMs) {
|
|
2448
|
+
return mutateGovernor(scope, nowMs, (state) => {
|
|
2449
|
+
if (!validGovernorId(intentId)) return { ok: false, reason: "corrupt" };
|
|
2450
|
+
if (state.intents[intentId]) {
|
|
2451
|
+
delete state.intents[intentId];
|
|
2452
|
+
return { value: { status: "cancelled", intentId } };
|
|
2453
|
+
}
|
|
2454
|
+
const reservationId = `reservation:${intentId}`;
|
|
2455
|
+
const reservation = state.reservations[reservationId];
|
|
2456
|
+
if (!reservation || reservation.status !== "scheduled") {
|
|
2457
|
+
return { ok: false, reason: "stale" };
|
|
2458
|
+
}
|
|
2459
|
+
delete state.reservations[reservationId];
|
|
2460
|
+
return { value: { status: "cancelled", intentId, reservationId } };
|
|
2461
|
+
});
|
|
2462
|
+
}
|
|
1039
2463
|
|
|
1040
|
-
|
|
1041
|
-
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
2464
|
+
function startReservation(scope, reservationId, nowMs) {
|
|
2465
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2466
|
+
const reservation = state.reservations[reservationId];
|
|
2467
|
+
const lease = reservation && state.leases[reservation.leaseId];
|
|
2468
|
+
if (!reservation || reservation.status !== "scheduled" || !lease || lease.expiresAt <= at) {
|
|
2469
|
+
if (reservation?.status === "scheduled") reservation.status = "cancelled";
|
|
2470
|
+
return { ok: false, reason: "stale", write: Boolean(reservation) };
|
|
2471
|
+
}
|
|
2472
|
+
if (reservation.notBefore > at) return { value: { status: "waiting", notBefore: reservation.notBefore } };
|
|
2473
|
+
if (state.probeClaim && state.probeClaim.leaseUntil > at) {
|
|
2474
|
+
return { value: { status: "waiting", reason: "probe", notBefore: state.probeClaim.leaseUntil } };
|
|
2475
|
+
}
|
|
2476
|
+
for (const resource of RATE_RESOURCES.filter((name) => reservation.costs[name] > 0)) {
|
|
2477
|
+
const budget = state.budgets[resource];
|
|
2478
|
+
if (
|
|
2479
|
+
!budget || at - budget.observedAt > BUDGET_SNAPSHOT_TTL_MS || at >= budget.resetMs ||
|
|
2480
|
+
state.epochs[resource] !== reservation.epochs[resource] ||
|
|
2481
|
+
(Number.isFinite(budget.blockUntil) && budget.blockUntil > at)
|
|
2482
|
+
) {
|
|
2483
|
+
reservation.status = "cancelled";
|
|
2484
|
+
return { ok: false, reason: "stale", write: true };
|
|
2485
|
+
}
|
|
2486
|
+
const charged = Object.values(state.reservations).reduce(
|
|
2487
|
+
(total, item) => total + reservationCost(item, resource, state.leases, at),
|
|
2488
|
+
0,
|
|
2489
|
+
);
|
|
2490
|
+
if (budget.remaining - resourceReserve(budget.limit) - charged < 0) {
|
|
2491
|
+
reservation.status = "cancelled";
|
|
2492
|
+
return { ok: false, reason: "stale", write: true };
|
|
2493
|
+
}
|
|
2494
|
+
}
|
|
2495
|
+
reservation.status = "started";
|
|
2496
|
+
reservation.startedAt = at;
|
|
2497
|
+
return { value: { status: "started", reservationId } };
|
|
2498
|
+
});
|
|
1046
2499
|
}
|
|
1047
2500
|
|
|
1048
|
-
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
|
|
1053
|
-
|
|
1054
|
-
|
|
1055
|
-
|
|
1056
|
-
|
|
1057
|
-
|
|
1058
|
-
|
|
1059
|
-
|
|
1060
|
-
|
|
1061
|
-
|
|
1062
|
-
|
|
1063
|
-
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
// be measured, or null before the first probe.
|
|
1067
|
-
//
|
|
1068
|
-
// The baseline only advances when the window was usable, so an unmeasurable
|
|
1069
|
-
// window is not thrown away -- it grows until it holds MIN_SAMPLE_CALLS of our
|
|
1070
|
-
// own calls, with both halves of the ratio still spanning the same stretch of
|
|
1071
|
-
// time. The cost is latency, not accuracy: a pane at the 60s cap needs two or
|
|
1072
|
-
// three probes rather than one to notice that the other panes have exited.
|
|
1073
|
-
function nextProbeWindow(prev, budget, spentTotal) {
|
|
1074
|
-
// A window reset makes `used` fall. The span before the reset cannot be
|
|
1075
|
-
// compared against the counter after it, so the window restarts here and this
|
|
1076
|
-
// cycle infers nothing -- the caller keeps the share it last measured, which
|
|
1077
|
-
// is the one direction that cannot spike the poll rate at the top of an hour.
|
|
1078
|
-
if (prev === null || budget.used < prev.used) {
|
|
1079
|
-
return { sample: null, next: { used: budget.used, spent: spentTotal } };
|
|
1080
|
-
}
|
|
1081
|
-
const sample = {
|
|
1082
|
-
globalUsed: budget.used - prev.used,
|
|
1083
|
-
myCalls: spentTotal - prev.spent,
|
|
1084
|
-
};
|
|
1085
|
-
return { sample, next: sampleIsUsable(sample) ? { used: budget.used, spent: spentTotal } : prev };
|
|
2501
|
+
function completeReservation(scope, reservationId, completion, nowMs) {
|
|
2502
|
+
return mutateGovernor(scope, nowMs, (state, at) => {
|
|
2503
|
+
const reservation = state.reservations[reservationId];
|
|
2504
|
+
if (!reservation || reservation.status !== "started" || !GOVERNOR_OUTCOMES.has(completion?.outcome)) {
|
|
2505
|
+
return { ok: false, reason: "stale" };
|
|
2506
|
+
}
|
|
2507
|
+
const measured = completion.outcome === "measured-success"
|
|
2508
|
+
? exactResourceCosts(completion.actualCost)
|
|
2509
|
+
: reservation.costs;
|
|
2510
|
+
if (!measured || RATE_RESOURCES.some((resource) => measured[resource] > reservation.costs[resource])) {
|
|
2511
|
+
return { ok: false, reason: "corrupt" };
|
|
2512
|
+
}
|
|
2513
|
+
reservation.status = "completed";
|
|
2514
|
+
reservation.completedAt = at;
|
|
2515
|
+
reservation.outcome = completion.outcome;
|
|
2516
|
+
reservation.actualCosts = { ...measured };
|
|
2517
|
+
return { value: { status: "completed", actualCosts: reservation.actualCosts } };
|
|
2518
|
+
});
|
|
1086
2519
|
}
|
|
1087
2520
|
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
|
|
1093
|
-
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
|
|
1102
|
-
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
2521
|
+
function recordResourceBlock(scope, resource, resetMs, reason) {
|
|
2522
|
+
return mutateGovernor(scope, undefined, (state, nowMs) => {
|
|
2523
|
+
const budget = state.budgets[resource];
|
|
2524
|
+
if (!budget || !RATE_RESOURCES.includes(resource) || !Number.isFinite(resetMs) || resetMs <= nowMs ||
|
|
2525
|
+
!GOVERNOR_BLOCK_REASONS.has(reason)) return { ok: false, reason: "corrupt" };
|
|
2526
|
+
budget.blockUntil = resetMs;
|
|
2527
|
+
budget.blockReason = reason;
|
|
2528
|
+
return { value: { resource, resetMs, reason } };
|
|
2529
|
+
});
|
|
2530
|
+
}
|
|
2531
|
+
|
|
2532
|
+
function releaseLease(scope, leaseId) {
|
|
2533
|
+
return mutateGovernor(scope, undefined, (state) => {
|
|
2534
|
+
if (!state.leases[leaseId]) return { ok: false, reason: "stale" };
|
|
2535
|
+
delete state.leases[leaseId];
|
|
2536
|
+
for (const [id, intent] of Object.entries(state.intents)) {
|
|
2537
|
+
if (intent.leaseId === leaseId) delete state.intents[id];
|
|
2538
|
+
}
|
|
2539
|
+
for (const [id, reservation] of Object.entries(state.reservations)) {
|
|
2540
|
+
if (reservation.leaseId === leaseId && reservation.status === "scheduled") delete state.reservations[id];
|
|
2541
|
+
}
|
|
2542
|
+
return { value: { released: leaseId } };
|
|
2543
|
+
});
|
|
2544
|
+
}
|
|
2545
|
+
|
|
2546
|
+
function inspectGovernor(scope, nowMs) {
|
|
2547
|
+
const requestedAt = scopeNow(scope, nowMs);
|
|
2548
|
+
if (!Number.isFinite(requestedAt) || requestedAt < 0) return { ok: false, reason: "corrupt" };
|
|
2549
|
+
return withGovernorLock(scope, () => {
|
|
2550
|
+
const at = governorEffectiveTime(scope, requestedAt);
|
|
2551
|
+
const loaded = readGovernorState(scope.path, at);
|
|
2552
|
+
return loaded;
|
|
2553
|
+
});
|
|
2554
|
+
}
|
|
1113
2555
|
|
|
1114
|
-
|
|
1115
|
-
|
|
1116
|
-
|
|
2556
|
+
function governorHealth(result, nowMs = Date.now()) {
|
|
2557
|
+
if (!result?.ok) return { status: "unavailable", leases: 0, resources: {} };
|
|
2558
|
+
const state = result.value;
|
|
2559
|
+
let status = "healthy";
|
|
2560
|
+
if (state.probeClaim?.leaseUntil > nowMs) status = "waiting for probe";
|
|
2561
|
+
else if (RATE_RESOURCES.some((resource) => !state.budgets[resource] || nowMs - state.budgets[resource].observedAt > BUDGET_SNAPSHOT_TTL_MS)) status = "stale";
|
|
2562
|
+
else if (RATE_RESOURCES.some((resource) => state.budgets[resource].blockUntil > nowMs)) status = "blocked";
|
|
2563
|
+
return {
|
|
2564
|
+
status,
|
|
2565
|
+
leases: Object.keys(state.leases).length,
|
|
2566
|
+
resources: Object.fromEntries(RATE_RESOURCES.flatMap((resource) => {
|
|
2567
|
+
const budget = state.budgets[resource];
|
|
2568
|
+
return budget ? [[resource, {
|
|
2569
|
+
reserve: resourceReserve(budget.limit),
|
|
2570
|
+
remaining: budget.remaining,
|
|
2571
|
+
resetMs: budget.resetMs,
|
|
2572
|
+
}]] : [];
|
|
2573
|
+
})),
|
|
2574
|
+
};
|
|
2575
|
+
}
|
|
1117
2576
|
|
|
1118
|
-
|
|
1119
|
-
|
|
1120
|
-
|
|
2577
|
+
async function retryGovernorMutation(run, { now, wait, deadline, signal }) {
|
|
2578
|
+
let result = run();
|
|
2579
|
+
while (!result.ok && result.reason === "busy" && !signal?.aborted && now() < deadline) {
|
|
2580
|
+
await wait(Math.min(100, Math.max(1, deadline - now())));
|
|
2581
|
+
result = run();
|
|
2582
|
+
}
|
|
2583
|
+
return result;
|
|
1121
2584
|
}
|
|
1122
2585
|
|
|
1123
|
-
|
|
1124
|
-
|
|
1125
|
-
|
|
1126
|
-
|
|
1127
|
-
|
|
2586
|
+
async function refreshSharedBudget(scope, leaseId, signal, {
|
|
2587
|
+
readBudgets = readRateBudgets,
|
|
2588
|
+
now = () => scopeNow(scope),
|
|
2589
|
+
wait = (ms) => new Promise((resolve) => setTimeout(resolve, ms)),
|
|
2590
|
+
inspect = inspectGovernor,
|
|
2591
|
+
renew = renewProbeClaim,
|
|
2592
|
+
publish = publishProbe,
|
|
2593
|
+
fail = failProbeClaim,
|
|
2594
|
+
} = {}) {
|
|
2595
|
+
let claim = claimProbe(scope, leaseId, now());
|
|
2596
|
+
if (!claim.ok) return claim;
|
|
2597
|
+
if (claim.value.status !== "claimed") {
|
|
2598
|
+
const startedAt = now();
|
|
2599
|
+
const deadline = Math.min(
|
|
2600
|
+
claim.value.leaseUntil ?? claim.value.nextAt ?? startedAt,
|
|
2601
|
+
startedAt + GOVERNOR_PUBLICATION_REINSPECT_MS,
|
|
2602
|
+
);
|
|
2603
|
+
let inspections = 0;
|
|
2604
|
+
while (!signal?.aborted && now() < deadline && inspections < 10) {
|
|
2605
|
+
inspections += 1;
|
|
2606
|
+
const snapshot = inspect(scope, now());
|
|
2607
|
+
if (!snapshot.ok) return snapshot;
|
|
2608
|
+
if (
|
|
2609
|
+
RATE_RESOURCES.every((resource) =>
|
|
2610
|
+
snapshot.value.budgets[resource] &&
|
|
2611
|
+
now() - snapshot.value.budgets[resource].observedAt <= BUDGET_SNAPSHOT_TTL_MS,
|
|
2612
|
+
) && snapshot.value.probeOutcome.status === "healthy"
|
|
2613
|
+
) {
|
|
2614
|
+
return { ok: true, value: { status: "published", budgets: snapshot.value.budgets } };
|
|
2615
|
+
}
|
|
2616
|
+
if (now() >= deadline || inspections >= 10) break;
|
|
2617
|
+
await wait(Math.min(100, Math.max(1, deadline - now())));
|
|
2618
|
+
}
|
|
2619
|
+
claim = claimProbe(scope, leaseId, now());
|
|
2620
|
+
if (!claim.ok || claim.value.status !== "claimed") return claim;
|
|
2621
|
+
}
|
|
2622
|
+
const { nonce, startedReservationIds } = claim.value;
|
|
2623
|
+
const drainUntil = now() + GOVERNOR_PROBE_DRAIN_MS;
|
|
2624
|
+
while (startedReservationIds.length > 0 && now() < drainUntil) {
|
|
2625
|
+
const snapshot = inspect(scope, now());
|
|
2626
|
+
if (!snapshot.ok) {
|
|
2627
|
+
failProbeClaim(scope, leaseId, nonce, now());
|
|
2628
|
+
return snapshot;
|
|
2629
|
+
}
|
|
2630
|
+
const stillStarted = startedReservationIds.some((id) => snapshot.value.reservations[id]?.status === "started");
|
|
2631
|
+
if (!stillStarted) break;
|
|
2632
|
+
if (signal?.aborted) {
|
|
2633
|
+
failProbeClaim(scope, leaseId, nonce, now());
|
|
2634
|
+
return { ok: false, reason: "stale" };
|
|
2635
|
+
}
|
|
2636
|
+
await wait(Math.min(100, Math.max(1, drainUntil - now())));
|
|
2637
|
+
}
|
|
2638
|
+
const renewalStartedAt = now();
|
|
2639
|
+
const renewalDeadline = Math.min(
|
|
2640
|
+
claim.value.leaseUntil,
|
|
2641
|
+
renewalStartedAt + GOVERNOR_PROBE_TRANSITION_MS,
|
|
2642
|
+
);
|
|
2643
|
+
const renewed = await retryGovernorMutation(
|
|
2644
|
+
() => renew(scope, leaseId, nonce, now()),
|
|
2645
|
+
{ now, wait, deadline: renewalDeadline, signal },
|
|
2646
|
+
);
|
|
2647
|
+
if (!renewed.ok) {
|
|
2648
|
+
failProbeClaim(scope, leaseId, nonce, now());
|
|
2649
|
+
return renewed;
|
|
2650
|
+
}
|
|
2651
|
+
let budgets;
|
|
2652
|
+
try {
|
|
2653
|
+
budgets = await readBudgets(signal, scope.host);
|
|
2654
|
+
} catch {
|
|
2655
|
+
budgets = null;
|
|
2656
|
+
}
|
|
2657
|
+
if (!budgets) {
|
|
2658
|
+
const failureDeadline = Math.min(
|
|
2659
|
+
renewed.value.leaseUntil ?? Number.POSITIVE_INFINITY,
|
|
2660
|
+
now() + GOVERNOR_PROBE_TRANSITION_MS,
|
|
2661
|
+
);
|
|
2662
|
+
await retryGovernorMutation(
|
|
2663
|
+
() => fail(scope, leaseId, nonce, now()),
|
|
2664
|
+
{ now, wait, deadline: failureDeadline, signal },
|
|
2665
|
+
);
|
|
2666
|
+
return { ok: false, reason: "stale" };
|
|
2667
|
+
}
|
|
2668
|
+
const transitionDeadline = Math.min(
|
|
2669
|
+
renewed.value.leaseUntil ?? Number.POSITIVE_INFINITY,
|
|
2670
|
+
now() + GOVERNOR_PROBE_TRANSITION_MS,
|
|
2671
|
+
);
|
|
2672
|
+
const published = await retryGovernorMutation(
|
|
2673
|
+
() => publish(scope, leaseId, nonce, budgets, now()),
|
|
2674
|
+
{ now, wait, deadline: transitionDeadline, signal },
|
|
2675
|
+
);
|
|
2676
|
+
if (!published.ok) {
|
|
2677
|
+
await retryGovernorMutation(
|
|
2678
|
+
() => fail(scope, leaseId, nonce, now()),
|
|
2679
|
+
{ now, wait, deadline: transitionDeadline, signal },
|
|
2680
|
+
);
|
|
2681
|
+
}
|
|
2682
|
+
return published;
|
|
1128
2683
|
}
|
|
1129
2684
|
|
|
1130
|
-
//
|
|
1131
|
-
//
|
|
1132
|
-
//
|
|
1133
|
-
|
|
1134
|
-
|
|
2685
|
+
// Whether a probe window holds enough completed shared calls for the ratio below to
|
|
2686
|
+
// mean anything. Named rather than inlined because the control law and the loop
|
|
2687
|
+
// that feeds it must agree on the answer: the law uses it to decide whether to
|
|
2688
|
+
// infer, and the loop uses it to decide whether the window may be closed.
|
|
2689
|
+
function externalSampleIsUsable(sample) {
|
|
2690
|
+
return (
|
|
2691
|
+
Boolean(sample) &&
|
|
2692
|
+
sample.sharedCompletedDelta >= MIN_SAMPLE_CALLS &&
|
|
2693
|
+
sample.globalUsedDelta > 0
|
|
2694
|
+
);
|
|
1135
2695
|
}
|
|
1136
2696
|
|
|
1137
2697
|
function resourcePerTick(table, activeKey) {
|
|
@@ -1142,26 +2702,6 @@ function resourcePerTick(table, activeKey) {
|
|
|
1142
2702
|
return cost;
|
|
1143
2703
|
}
|
|
1144
2704
|
|
|
1145
|
-
function nextBudgetTargets({ budgets, samples, shares, previous, activeKey, floorMs, nowMs }) {
|
|
1146
|
-
const targets = { core: floorMs, graphql: floorMs, ...previous };
|
|
1147
|
-
for (const [resource, table] of [
|
|
1148
|
-
["core", REST_PER_FETCH],
|
|
1149
|
-
["graphql", GRAPHQL_PER_FETCH],
|
|
1150
|
-
]) {
|
|
1151
|
-
const callsPerTick = resourcePerTick(table, activeKey);
|
|
1152
|
-
if (callsPerTick <= 0 || !budgets?.[resource]) continue;
|
|
1153
|
-
targets[resource] = adaptiveRefreshMs({
|
|
1154
|
-
budget: budgets[resource],
|
|
1155
|
-
sample: samples?.[resource] ?? null,
|
|
1156
|
-
lastShare: shares?.[resource] ?? 1,
|
|
1157
|
-
floorMs,
|
|
1158
|
-
nowMs,
|
|
1159
|
-
restPerTick: callsPerTick,
|
|
1160
|
-
});
|
|
1161
|
-
}
|
|
1162
|
-
return { targets, targetMs: Math.max(floorMs, ...Object.values(targets)) };
|
|
1163
|
-
}
|
|
1164
|
-
|
|
1165
2705
|
function alertArgs(source, path = source.path) {
|
|
1166
2706
|
return ["api", apiPath(path), ...apiHostArgs(), "--jq", source.jq];
|
|
1167
2707
|
}
|
|
@@ -1199,32 +2739,17 @@ function mergeAlertRows(groups) {
|
|
|
1199
2739
|
// mid-session is picked up within the hour.
|
|
1200
2740
|
const alertBackoff = new Map();
|
|
1201
2741
|
|
|
1202
|
-
// REST and GraphQL calls this process has spent, ever. In-memory and per-process,
|
|
1203
|
-
// in the same spirit as alertBackoff above: lost on exit, and nothing depends on
|
|
1204
|
-
// it surviving. Each counter is compared only against its matching resource
|
|
1205
|
-
// between two budget probes, so neither needs persistence or an extra window.
|
|
1206
|
-
//
|
|
1207
|
-
// Deliberately incomplete: openItem, preflight and resolveFailureContext also
|
|
1208
|
-
// spend, and are not counted, because they are occasional rather than periodic.
|
|
1209
|
-
// Under-reporting our own spend makes the inferred share *larger* and so
|
|
1210
|
-
// throttles slightly harder than strictly necessary -- the safe direction to err
|
|
1211
|
-
// in, which is why the imprecision is tolerated rather than chased.
|
|
1212
|
-
const spentTotal = { core: 0, graphql: 0 };
|
|
1213
|
-
|
|
1214
2742
|
function backoffActive(key, now) {
|
|
1215
2743
|
const state = alertBackoff.get(key);
|
|
1216
2744
|
return Boolean(state) && now < state.until;
|
|
1217
2745
|
}
|
|
1218
2746
|
|
|
1219
|
-
// Which ladder each verdict takes.
|
|
1220
|
-
//
|
|
1221
|
-
// comes back the moment it does. "rate-limited" used to be absent for the same
|
|
1222
|
-
// stated reason, which had it backwards -- see RATE_LIMIT_RETRY_MS.
|
|
2747
|
+
// Which local ladder each verdict takes. Shared rate limits are held by the
|
|
2748
|
+
// account governor instead of a process-local retry timer.
|
|
1223
2749
|
const FAILURE_LADDER = {
|
|
1224
2750
|
"no-remote": BACKOFF_STEPS_MS,
|
|
1225
2751
|
unavailable: BACKOFF_STEPS_MS,
|
|
1226
2752
|
"auth-problem": AUTH_RETRY_MS,
|
|
1227
|
-
"rate-limited": RATE_LIMIT_RETRY_MS,
|
|
1228
2753
|
};
|
|
1229
2754
|
|
|
1230
2755
|
// The ladder is a parameter rather than a second near-identical function,
|
|
@@ -1247,6 +2772,12 @@ function forcedBackoffKeys(key) {
|
|
|
1247
2772
|
];
|
|
1248
2773
|
}
|
|
1249
2774
|
|
|
2775
|
+
function clearForcedBackoffAfterStart(key, force, status, clear = clearBackoff) {
|
|
2776
|
+
if (!force || status !== "started") return false;
|
|
2777
|
+
for (const backoffKey of forcedBackoffKeys(key)) clear(backoffKey);
|
|
2778
|
+
return true;
|
|
2779
|
+
}
|
|
2780
|
+
|
|
1250
2781
|
async function fetchAlertSource(source, signal, now) {
|
|
1251
2782
|
if (backoffActive(source.key, now)) {
|
|
1252
2783
|
const { note, verdict } = alertBackoff.get(source.key);
|
|
@@ -1257,19 +2788,20 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
1257
2788
|
// the meter is fed by the fetchers rather than by counting ticks.
|
|
1258
2789
|
return {
|
|
1259
2790
|
raw: `backoff:${note}`,
|
|
1260
|
-
|
|
2791
|
+
completedCalls: 0,
|
|
2792
|
+
verdict,
|
|
1261
2793
|
parse: () => ({ alerts: [], note, verdict, truncated: false }),
|
|
1262
2794
|
};
|
|
1263
2795
|
}
|
|
1264
|
-
let
|
|
2796
|
+
let completedCalls = 0;
|
|
1265
2797
|
try {
|
|
1266
2798
|
const requests = alertRequestArgs(source);
|
|
1267
2799
|
const payloads = [];
|
|
1268
2800
|
const groups = [];
|
|
1269
2801
|
for (const [index, args] of requests.entries()) {
|
|
1270
2802
|
if (index > 0 && !shouldFetchAlertPriorityLanes(groups[0]?.length ?? 0)) break;
|
|
1271
|
-
|
|
1272
|
-
const payload = await runGh(args, { signal });
|
|
2803
|
+
completedCalls += 1;
|
|
2804
|
+
const payload = await runGh(args, { signal, operation: "tab:security-endpoint" });
|
|
1273
2805
|
payloads.push(payload);
|
|
1274
2806
|
groups.push(JSON.parse(payload).filter((alert) => alert.state === "open"));
|
|
1275
2807
|
}
|
|
@@ -1277,7 +2809,8 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
1277
2809
|
clearBackoff(source.key);
|
|
1278
2810
|
return {
|
|
1279
2811
|
raw,
|
|
1280
|
-
|
|
2812
|
+
completedCalls,
|
|
2813
|
+
verdict: "ok",
|
|
1281
2814
|
parse: () => {
|
|
1282
2815
|
const rows = mergeAlertRows(groups);
|
|
1283
2816
|
return {
|
|
@@ -1304,7 +2837,8 @@ async function fetchAlertSource(source, signal, now) {
|
|
|
1304
2837
|
raw: `unavailable:${note}`,
|
|
1305
2838
|
// A failed call still bills: the request reached GitHub and was counted
|
|
1306
2839
|
// whether it returned data, `[]`, or a 403.
|
|
1307
|
-
|
|
2840
|
+
completedCalls,
|
|
2841
|
+
verdict,
|
|
1308
2842
|
parse: () => ({ alerts: [], note, verdict, truncated: false }),
|
|
1309
2843
|
};
|
|
1310
2844
|
}
|
|
@@ -1341,8 +2875,10 @@ async function fetchSecurity(signal) {
|
|
|
1341
2875
|
// Summed from what each source reported rather than assumed to be
|
|
1342
2876
|
// ALERT_SOURCES.length: a source held off by backoff spawned nothing and
|
|
1343
2877
|
// must not be billed for it.
|
|
1344
|
-
restSpent: parts.reduce((
|
|
2878
|
+
restSpent: parts.reduce((total, part) => total + part.completedCalls, 0),
|
|
1345
2879
|
graphqlSpent: 0,
|
|
2880
|
+
measuredSuccess: parts.every((part) => part.verdict === "ok"),
|
|
2881
|
+
rateLimited: parts.some((part) => part.verdict === "rate-limited"),
|
|
1346
2882
|
parse: () => {
|
|
1347
2883
|
const parsed = parts.map((p) => p.parse());
|
|
1348
2884
|
const alerts = parsed.flatMap((p) => p.alerts);
|
|
@@ -1407,11 +2943,21 @@ const OPENABLE = { actions: "run", issues: "issue", prs: "pr" };
|
|
|
1407
2943
|
// almost never a valid databaseId elsewhere and 404s. `gh issue view` and
|
|
1408
2944
|
// `gh pr view` are the opposite: they take the issue/PR number, and neither
|
|
1409
2945
|
// row shape carries a databaseId to prefer instead.
|
|
1410
|
-
async function openInBrowser(tabKey, item, signal) {
|
|
2946
|
+
async function openInBrowser(tabKey, item, signal, governor = null, { run = runGh } = {}) {
|
|
1411
2947
|
const kind = pick(OPENABLE, tabKey, null);
|
|
1412
2948
|
const id = kind === "run" ? (item?.databaseId ?? item?.number) : (item?.number ?? item?.databaseId);
|
|
1413
2949
|
if (!kind || id == null) return;
|
|
1414
|
-
|
|
2950
|
+
if (!governor) throw new Error("API budget unavailable; refresh and try again");
|
|
2951
|
+
const admitted = await runAdmittedOperation({
|
|
2952
|
+
...governor,
|
|
2953
|
+
operation: `open:${tabKey}`,
|
|
2954
|
+
signal,
|
|
2955
|
+
run: (admittedSignal) => run([kind, "view", ...repoArgs(), String(id), "--web"], {
|
|
2956
|
+
signal: admittedSignal,
|
|
2957
|
+
operation: `open:${tabKey}`,
|
|
2958
|
+
}),
|
|
2959
|
+
});
|
|
2960
|
+
if (!admitted.ok) throw admitted.error;
|
|
1415
2961
|
}
|
|
1416
2962
|
|
|
1417
2963
|
function createOpenRequestRegistry() {
|
|
@@ -1566,9 +3112,9 @@ function envValue(name) {
|
|
|
1566
3112
|
// gh writes some of this to stdout and some to stderr depending on version and
|
|
1567
3113
|
// on whether it succeeded, and a non-zero exit is itself worth reporting rather
|
|
1568
3114
|
// than throwing. Both streams, whatever happened.
|
|
1569
|
-
async function captureGh(args) {
|
|
3115
|
+
async function captureGh(args, operation) {
|
|
1570
3116
|
try {
|
|
1571
|
-
return (await runGh(args)).trim();
|
|
3117
|
+
return (await runGh(args, { operation })).trim();
|
|
1572
3118
|
} catch (err) {
|
|
1573
3119
|
const both = `${err?.stdout ?? ""}${err?.stderr ?? ""}`.trim();
|
|
1574
3120
|
return both || shortErr(err);
|
|
@@ -1577,10 +3123,10 @@ async function captureGh(args) {
|
|
|
1577
3123
|
|
|
1578
3124
|
const PROBE_STDERR_LIMIT = 400;
|
|
1579
3125
|
|
|
1580
|
-
async function probe(name, args) {
|
|
3126
|
+
async function probe(name, args, operation) {
|
|
1581
3127
|
const startedAt = Date.now();
|
|
1582
3128
|
try {
|
|
1583
|
-
const stdout = await runGh(args);
|
|
3129
|
+
const stdout = await runGh(args, { operation });
|
|
1584
3130
|
return { name, args, ms: Date.now() - startedAt, bytes: stdout.length, classified: "ok" };
|
|
1585
3131
|
} catch (err) {
|
|
1586
3132
|
return {
|
|
@@ -1626,6 +3172,22 @@ async function gitRemote() {
|
|
|
1626
3172
|
}
|
|
1627
3173
|
}
|
|
1628
3174
|
|
|
3175
|
+
async function gitRemoteUrls() {
|
|
3176
|
+
try {
|
|
3177
|
+
const { stdout } = await execFileAsync("git", ["remote"], { timeout: GH_TIMEOUT_MS });
|
|
3178
|
+
const names = stdout.split(/\r?\n/).map((name) => name.trim()).filter(Boolean);
|
|
3179
|
+
const settled = await Promise.allSettled(names.map(async (name) => {
|
|
3180
|
+
const result = await execFileAsync("git", ["remote", "get-url", "--all", name], {
|
|
3181
|
+
timeout: GH_TIMEOUT_MS,
|
|
3182
|
+
});
|
|
3183
|
+
return result.stdout.split(/\r?\n/).map((url) => url.trim()).filter(Boolean);
|
|
3184
|
+
}));
|
|
3185
|
+
return settled.flatMap((result) => result.status === "fulfilled" ? result.value : []);
|
|
3186
|
+
} catch {
|
|
3187
|
+
return [];
|
|
3188
|
+
}
|
|
3189
|
+
}
|
|
3190
|
+
|
|
1629
3191
|
function targetSource() {
|
|
1630
3192
|
if (runtime.repo) return "flag";
|
|
1631
3193
|
if (process.env.GH_REPO) return "GH_REPO";
|
|
@@ -1645,18 +3207,25 @@ function targetSource() {
|
|
|
1645
3207
|
// measures as free (verified: delta 0), so this is safe to run on a diagnostic
|
|
1646
3208
|
// path. GHES tenants can be configured with a different ceiling, which is
|
|
1647
3209
|
// exactly why this reports the server's own numbers rather than asserting 5,000.
|
|
1648
|
-
async function
|
|
3210
|
+
async function readRateLimitResources(signal, host = effectiveRuntimeHost()) {
|
|
3211
|
+
const raw = await runGh(["api", "rate_limit", ...apiHostArgs(host)], {
|
|
3212
|
+
signal,
|
|
3213
|
+
operation: "rate-limit",
|
|
3214
|
+
});
|
|
3215
|
+
return JSON.parse(raw)?.resources;
|
|
3216
|
+
}
|
|
3217
|
+
|
|
3218
|
+
async function rateBudget(preloadedResources = null) {
|
|
1649
3219
|
try {
|
|
1650
3220
|
// Host-routed like every other `gh api` call: a budget is per token *per
|
|
1651
3221
|
// server*, so on a GHES tenant the github.com numbers are not merely stale,
|
|
1652
3222
|
// they belong to a different limit entirely.
|
|
1653
|
-
const
|
|
1654
|
-
const { resources } = JSON.parse(raw);
|
|
3223
|
+
const resources = normalizeRateBudgets(preloadedResources ?? await readRateLimitResources());
|
|
1655
3224
|
// formatAge() is for timestamps in the past and returns "-" for a future one,
|
|
1656
3225
|
// so the reset is rendered as a forward interval instead.
|
|
1657
3226
|
const fmt = (r) => {
|
|
1658
3227
|
if (!r) return "(absent)";
|
|
1659
|
-
const inMs = r.
|
|
3228
|
+
const inMs = r.resetMs - Date.now();
|
|
1660
3229
|
const resets = inMs > 0 ? `resets in ${formatDuration(inMs)}` : "reset due";
|
|
1661
3230
|
return `${r.remaining}/${r.limit} left, ${resets}`;
|
|
1662
3231
|
};
|
|
@@ -1666,8 +3235,7 @@ async function rateBudget() {
|
|
|
1666
3235
|
}
|
|
1667
3236
|
}
|
|
1668
3237
|
|
|
1669
|
-
// The same probe as rateBudget, but returning both resources as data
|
|
1670
|
-
// sentences, because the adaptive loop has to do arithmetic on each budget.
|
|
3238
|
+
// The same probe as rateBudget, but returning both resources as protocol data.
|
|
1671
3239
|
// rateBudget keeps its display shape for the --doctor path, where strings are
|
|
1672
3240
|
// the product.
|
|
1673
3241
|
//
|
|
@@ -1692,31 +3260,33 @@ function normalizeRateBudget(resource) {
|
|
|
1692
3260
|
};
|
|
1693
3261
|
}
|
|
1694
3262
|
|
|
1695
|
-
|
|
3263
|
+
function normalizeRateBudgets(resources) {
|
|
3264
|
+
return Object.fromEntries(RATE_RESOURCES.map((resource) => [
|
|
3265
|
+
resource,
|
|
3266
|
+
normalizeRateBudget(resources?.[resource]),
|
|
3267
|
+
]));
|
|
3268
|
+
}
|
|
3269
|
+
|
|
3270
|
+
async function readRateBudgets(signal, host = effectiveRuntimeHost()) {
|
|
1696
3271
|
try {
|
|
1697
3272
|
// apiHostArgs() for the same reason the alert endpoints carry it: --repo
|
|
1698
3273
|
// host/owner/name sets runtime.host without setting GH_HOST, so without the
|
|
1699
3274
|
// flag this reads github.com's budget while the pane spends against the
|
|
1700
3275
|
// tenant -- and then throttles, or fails to, against an unrelated number.
|
|
1701
|
-
const
|
|
1702
|
-
|
|
1703
|
-
return {
|
|
1704
|
-
core: normalizeRateBudget(resources?.core),
|
|
1705
|
-
graphql: normalizeRateBudget(resources?.graphql),
|
|
1706
|
-
};
|
|
3276
|
+
const resources = await readRateLimitResources(signal, host);
|
|
3277
|
+
return normalizeRateBudgets(resources);
|
|
1707
3278
|
} catch {
|
|
1708
|
-
//
|
|
1709
|
-
//
|
|
3279
|
+
// The caller publishes a failed probe outcome and keeps data admission
|
|
3280
|
+
// closed. A missing sample must never fall back to the configured floor.
|
|
1710
3281
|
return null;
|
|
1711
3282
|
}
|
|
1712
3283
|
}
|
|
1713
3284
|
|
|
1714
|
-
//
|
|
1715
|
-
//
|
|
1716
|
-
//
|
|
1717
|
-
//
|
|
1718
|
-
// and GRAPHQL_PER_FETCH, which
|
|
1719
|
-
// two copies of them would diverge the first time one was fixed.
|
|
3285
|
+
// Conservative unpaced demand at this configuration's floor, derived from the
|
|
3286
|
+
// same constants the scheduler uses rather than from a number written down once
|
|
3287
|
+
// and left to rot. The governor can admit less demand; this projection explains
|
|
3288
|
+
// pressure, not actual granted spend. The per-fetch prices come from
|
|
3289
|
+
// REST_PER_FETCH and GRAPHQL_PER_FETCH, which also feed the operation registry.
|
|
1720
3290
|
function projectedHourlyCost(activeKey) {
|
|
1721
3291
|
const perHour = 3_600_000 / runtime.refreshMs;
|
|
1722
3292
|
return {
|
|
@@ -1725,34 +3295,134 @@ function projectedHourlyCost(activeKey) {
|
|
|
1725
3295
|
};
|
|
1726
3296
|
}
|
|
1727
3297
|
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1732
|
-
|
|
1733
|
-
|
|
1734
|
-
["Repository access", repoContextArgs()],
|
|
1735
|
-
["Actions (run list)", actionsArgs(MIN_RUN_LIMIT)],
|
|
1736
|
-
["Issues (issue list)", issuesArgs()],
|
|
1737
|
-
["Pull requests (pr list)", prsArgs()],
|
|
3298
|
+
function doctorProbePlan() {
|
|
3299
|
+
return [
|
|
3300
|
+
["Repository access", repoContextArgs(), "doctor:repository"],
|
|
3301
|
+
["Actions (run list)", actionsArgs(MIN_RUN_LIMIT), "doctor:actions"],
|
|
3302
|
+
["Issues (issue list)", issuesArgs(), "doctor:issues"],
|
|
3303
|
+
["Pull requests (pr list)", prsArgs(), "doctor:prs"],
|
|
1738
3304
|
...ALERT_SOURCES.flatMap((source) =>
|
|
1739
3305
|
alertRequestArgs(source).map((args, index) => [
|
|
1740
3306
|
index === 0 ? source.name : `${source.name} (priority ${index})`,
|
|
1741
3307
|
args,
|
|
3308
|
+
"doctor:security-endpoint",
|
|
1742
3309
|
]),
|
|
1743
3310
|
),
|
|
1744
3311
|
];
|
|
3312
|
+
}
|
|
3313
|
+
|
|
3314
|
+
async function mapAllSettledBounded(items, limit, map) {
|
|
3315
|
+
const settled = Array(items.length);
|
|
3316
|
+
let nextIndex = 0;
|
|
3317
|
+
const worker = async () => {
|
|
3318
|
+
while (nextIndex < items.length) {
|
|
3319
|
+
const index = nextIndex;
|
|
3320
|
+
nextIndex += 1;
|
|
3321
|
+
[settled[index]] = await Promise.allSettled([map(items[index], index)]);
|
|
3322
|
+
}
|
|
3323
|
+
};
|
|
3324
|
+
await Promise.allSettled(Array.from(
|
|
3325
|
+
{ length: Math.min(Math.max(1, limit), items.length) },
|
|
3326
|
+
() => worker(),
|
|
3327
|
+
));
|
|
3328
|
+
return settled;
|
|
3329
|
+
}
|
|
3330
|
+
|
|
3331
|
+
function skippedDoctorProbe(name, args, admitted) {
|
|
3332
|
+
return {
|
|
3333
|
+
name,
|
|
3334
|
+
args,
|
|
3335
|
+
skipped: true,
|
|
3336
|
+
classified: "skipped",
|
|
3337
|
+
skipReason: admitted?.value?.resetMs
|
|
3338
|
+
? `budget paused; reset ${new Date(admitted.value.resetMs).toISOString()}`
|
|
3339
|
+
: admitted?.value?.notBefore
|
|
3340
|
+
? `next safe slot ${new Date(admitted.value.notBefore).toISOString()}`
|
|
3341
|
+
: "budget unavailable",
|
|
3342
|
+
};
|
|
3343
|
+
}
|
|
3344
|
+
|
|
3345
|
+
async function runDoctor() {
|
|
3346
|
+
// A live backoff would make an alert probe silently skip and report nothing,
|
|
3347
|
+
// which is the opposite of what a diagnostic run is for.
|
|
3348
|
+
alertBackoff.clear();
|
|
3349
|
+
|
|
3350
|
+
const probes = doctorProbePlan();
|
|
1745
3351
|
|
|
1746
|
-
//
|
|
1747
|
-
//
|
|
1748
|
-
|
|
1749
|
-
|
|
1750
|
-
captureGh(["
|
|
1751
|
-
captureGh(["auth", "status"]),
|
|
3352
|
+
// Free checks run first. In particular the one rate_limit read both renders
|
|
3353
|
+
// the budget and seeds admission; diagnostics never spend before that proof.
|
|
3354
|
+
const [ghVersion, authStatus, remote, remoteUrls] = await Promise.all([
|
|
3355
|
+
captureGh(["--version"], "version"),
|
|
3356
|
+
captureGh(["auth", "status"], "auth-status"),
|
|
1752
3357
|
gitRemote(),
|
|
1753
|
-
|
|
1754
|
-
Promise.all(probes.map(([name, args]) => probe(name, args))),
|
|
3358
|
+
gitRemoteUrls(),
|
|
1755
3359
|
]);
|
|
3360
|
+
const effectiveHost = effectiveRuntimeHost({ remoteUrls });
|
|
3361
|
+
const resources = await readRateLimitResources(undefined, effectiveHost).catch(() => null);
|
|
3362
|
+
const budget = resources
|
|
3363
|
+
? await rateBudget(resources)
|
|
3364
|
+
: { core: "unavailable", graphql: "unavailable" };
|
|
3365
|
+
const scopeResult = createGovernorScope({ effectiveHost });
|
|
3366
|
+
let governorResult = scopeResult;
|
|
3367
|
+
let results = [];
|
|
3368
|
+
if (scopeResult.ok) {
|
|
3369
|
+
const scope = scopeResult.value;
|
|
3370
|
+
const leaseId = governorId();
|
|
3371
|
+
const nowMs = Date.now();
|
|
3372
|
+
const registered = registerLease(scope, {
|
|
3373
|
+
id: leaseId,
|
|
3374
|
+
expiresAt: nowMs + GOVERNOR_LEASE_TTL_MS,
|
|
3375
|
+
floorMs: runtime.refreshMs,
|
|
3376
|
+
activeTab: "actions",
|
|
3377
|
+
phaseSeed: { seed: leaseId, registeredAt: nowMs },
|
|
3378
|
+
demand: { core: 6, graphql: 2 },
|
|
3379
|
+
});
|
|
3380
|
+
try {
|
|
3381
|
+
const normalized = resources && normalizeRateBudgets(resources);
|
|
3382
|
+
if (registered.ok && normalized?.core && normalized?.graphql) {
|
|
3383
|
+
const claim = claimProbe(scope, leaseId, nowMs);
|
|
3384
|
+
if (claim.value?.status === "claimed") {
|
|
3385
|
+
publishProbe(scope, leaseId, claim.value.nonce, normalized, nowMs);
|
|
3386
|
+
}
|
|
3387
|
+
}
|
|
3388
|
+
const admissionAt = Date.now();
|
|
3389
|
+
const admittedProbes = probes.map(([name, args, operation], index) => {
|
|
3390
|
+
const admitted = registered.ok
|
|
3391
|
+
? admitGovernorOperation(scope, leaseId, operation, "diagnostic", admissionAt)
|
|
3392
|
+
: registered;
|
|
3393
|
+
if (!admitted?.ok || admitted.value.status !== "started") {
|
|
3394
|
+
return { index, skipped: skippedDoctorProbe(name, args, admitted) };
|
|
3395
|
+
}
|
|
3396
|
+
return { index, name, args, operation, reservationId: admitted.value.reservationId };
|
|
3397
|
+
});
|
|
3398
|
+
results = admittedProbes.map((item) => item.skipped ?? null);
|
|
3399
|
+
const runnable = admittedProbes.filter((item) => !item.skipped);
|
|
3400
|
+
const settled = await mapAllSettledBounded(runnable, 4, async (item) => {
|
|
3401
|
+
const result = await probe(item.name, item.args, item.operation);
|
|
3402
|
+
completeReservation(scope, item.reservationId, result.failed
|
|
3403
|
+
? { outcome: "rejected" }
|
|
3404
|
+
: { outcome: "measured-success", actualCost: operationCost(item.operation) }, Date.now());
|
|
3405
|
+
return result;
|
|
3406
|
+
});
|
|
3407
|
+
settled.forEach((outcome, index) => {
|
|
3408
|
+
const item = runnable[index];
|
|
3409
|
+
results[item.index] = outcome.status === "fulfilled"
|
|
3410
|
+
? outcome.value
|
|
3411
|
+
: {
|
|
3412
|
+
name: item.name,
|
|
3413
|
+
args: item.args,
|
|
3414
|
+
failed: true,
|
|
3415
|
+
classified: "other",
|
|
3416
|
+
stderr: shortErr(outcome.reason),
|
|
3417
|
+
};
|
|
3418
|
+
});
|
|
3419
|
+
} finally {
|
|
3420
|
+
const cleanup = { ...scope, identityProvider: null };
|
|
3421
|
+
releaseLease(cleanup, leaseId);
|
|
3422
|
+
governorResult = inspectGovernor(cleanup, Date.now());
|
|
3423
|
+
}
|
|
3424
|
+
}
|
|
3425
|
+
const governor = governorHealth(governorResult, Date.now());
|
|
1756
3426
|
|
|
1757
3427
|
const lines = [
|
|
1758
3428
|
"gh-glance doctor",
|
|
@@ -1766,7 +3436,7 @@ async function runDoctor() {
|
|
|
1766
3436
|
"",
|
|
1767
3437
|
...section("Repository target"),
|
|
1768
3438
|
field("source", targetSource()),
|
|
1769
|
-
field("host",
|
|
3439
|
+
field("host", effectiveHost ?? "(unresolved)"),
|
|
1770
3440
|
field("slug", runtime.repo ?? "(inferred from the working directory)"),
|
|
1771
3441
|
field("git remote", remote),
|
|
1772
3442
|
"",
|
|
@@ -1782,6 +3452,19 @@ async function runDoctor() {
|
|
|
1782
3452
|
})(),
|
|
1783
3453
|
),
|
|
1784
3454
|
"",
|
|
3455
|
+
...section("API governor"),
|
|
3456
|
+
field("status", governor.status),
|
|
3457
|
+
field("live leases", governor.leases),
|
|
3458
|
+
...RATE_RESOURCES.map((resource) => {
|
|
3459
|
+
const detail = governor.resources[resource];
|
|
3460
|
+
return field(
|
|
3461
|
+
resource,
|
|
3462
|
+
detail
|
|
3463
|
+
? `${detail.remaining} remaining, ${detail.reserve} reserved, reset ${new Date(detail.resetMs).toISOString()}`
|
|
3464
|
+
: "unavailable",
|
|
3465
|
+
);
|
|
3466
|
+
}),
|
|
3467
|
+
"",
|
|
1785
3468
|
...section("Environment"),
|
|
1786
3469
|
...doctorEnvNames().map((name) => field(name, envValue(name))),
|
|
1787
3470
|
"",
|
|
@@ -1794,7 +3477,9 @@ async function runDoctor() {
|
|
|
1794
3477
|
probeLine("argv", `gh ${result.args.join(" ")}`);
|
|
1795
3478
|
probeLine(
|
|
1796
3479
|
"outcome",
|
|
1797
|
-
result.
|
|
3480
|
+
result.skipped
|
|
3481
|
+
? `SKIPPED (${result.skipReason})`
|
|
3482
|
+
: result.failed ? `FAILED in ${result.ms}ms` : `ok ${result.bytes}B in ${result.ms}ms`,
|
|
1798
3483
|
);
|
|
1799
3484
|
if (result.http) probeLine("http", result.http);
|
|
1800
3485
|
probeLine("classified", result.classified);
|
|
@@ -1975,7 +3660,7 @@ const KEY_TABLE = [
|
|
|
1975
3660
|
["Up / Down, j / k", "Move the cursor between rows"],
|
|
1976
3661
|
["PgUp / PgDn", "Move a page at a time"],
|
|
1977
3662
|
["Enter", "Open the selected item or accept a prompt"],
|
|
1978
|
-
["r", "Refresh the current tab
|
|
3663
|
+
["r", "Refresh the current tab when a safe grant is available"],
|
|
1979
3664
|
["w", "Adjust table column widths"],
|
|
1980
3665
|
["?", "Show the keys (any key closes it)"],
|
|
1981
3666
|
["q / Esc / Ctrl+C", "Quit (Esc leaves width mode)"],
|
|
@@ -1999,7 +3684,7 @@ const HELP = `gh-glance ${version} -- a live-refreshing GitHub dashboard for a n
|
|
|
1999
3684
|
Usage:
|
|
2000
3685
|
gh-glance Run the dashboard in the current repository
|
|
2001
3686
|
gh-glance --repo owner/name Watch a specific repository instead
|
|
2002
|
-
gh-glance --refresh 15
|
|
3687
|
+
gh-glance --refresh 15 Set a 15-second active-tab poll floor
|
|
2003
3688
|
gh-glance --tab security Start on a specific tab
|
|
2004
3689
|
gh-glance --verbose 2>log Log every gh call to a file (see below)
|
|
2005
3690
|
gh-glance --doctor Print a diagnostic report and exit
|
|
@@ -2014,16 +3699,18 @@ Options:
|
|
|
2014
3699
|
data-residency tenant (e.g. tenant.ghe.com/acme/api)
|
|
2015
3700
|
and is unnecessary when running inside a clone of
|
|
2016
3701
|
that repository.
|
|
2017
|
-
--refresh <seconds>
|
|
2018
|
-
default ${REFRESH_MS / 1000}).
|
|
2019
|
-
|
|
3702
|
+
--refresh <seconds> Minimum active-tab poll interval (${MIN_REFRESH_SECONDS}-${MAX_REFRESH_SECONDS},
|
|
3703
|
+
default ${REFRESH_MS / 1000}). Safe shared grants may
|
|
3704
|
+
run later; each background tab is considered about
|
|
3705
|
+
every ${BACKGROUND_EVERY} floors.
|
|
2020
3706
|
--tab <name> Tab to start on: ${TAB_KEYS.join(", ")}.
|
|
2021
3707
|
--verbose Write one line per gh invocation to stderr. stderr
|
|
2022
3708
|
must be redirected -- writing it to the terminal
|
|
2023
3709
|
would draw over the dashboard, so this refuses to
|
|
2024
3710
|
start otherwise.
|
|
2025
3711
|
--doctor Gather versions, authenticated hosts, the resolved
|
|
2026
|
-
repo target
|
|
3712
|
+
repo target, API governor health and one safely
|
|
3713
|
+
admitted probe per endpoint, then exit.
|
|
2027
3714
|
Safe to redirect to a file and share -- tokens are
|
|
2028
3715
|
never printed, proxy credentials are stripped, and
|
|
2029
3716
|
no response bodies are included.
|
|
@@ -2032,11 +3719,14 @@ Run it from inside a locally cloned GitHub repository; the repo is inferred
|
|
|
2032
3719
|
from the git remote, the same way \`gh\` does it. Requires the \`gh\` CLI
|
|
2033
3720
|
(2.20 or newer), authenticated via \`gh auth login\`.
|
|
2034
3721
|
|
|
2035
|
-
|
|
2036
|
-
${(REFRESH_MS * BACKGROUND_EVERY) / 1000}s
|
|
3722
|
+
On a healthy single pane, the active tab is considered every ${REFRESH_MS / 1000}s and each
|
|
3723
|
+
background tab about every ${(REFRESH_MS * BACKGROUND_EVERY) / 1000}s. Quota-consuming calls still
|
|
3724
|
+
need a shared, resource-specific grant that preserves a hard reserve. Waiting
|
|
3725
|
+
and manual refresh do not bypass that safety check.
|
|
2037
3726
|
|
|
2038
|
-
|
|
2039
|
-
GH_GLANCE_ICONS=unicode for
|
|
3727
|
+
Row icons are GitHub Octicons and need a Nerd Font. Without one, set
|
|
3728
|
+
GH_GLANCE_ICONS=unicode for Unicode status glyphs and text row substitutes, or
|
|
3729
|
+
GH_GLANCE_ICONS=ascii for ASCII-only status and row icons.
|
|
2040
3730
|
|
|
2041
3731
|
Keys:
|
|
2042
3732
|
${keyTableLines()
|
|
@@ -2046,15 +3736,17 @@ ${keyTableLines()
|
|
|
2046
3736
|
The cursor clears itself after 60s with no movement.
|
|
2047
3737
|
|
|
2048
3738
|
Environment:
|
|
2049
|
-
GH_REPO=owner/name
|
|
2050
|
-
GH_HOST=host GitHub Enterprise or EMU host
|
|
2051
|
-
A
|
|
2052
|
-
|
|
3739
|
+
GH_REPO=[host/]owner/name Watch a specific repository (--repo takes precedence)
|
|
3740
|
+
GH_HOST=host GitHub Enterprise or EMU host for every call and
|
|
3741
|
+
its account governor. A qualified GH_REPO also
|
|
3742
|
+
supplies the host; an unqualified one means github.com.
|
|
2053
3743
|
GH_GLANCE_REFRESH=<seconds>
|
|
2054
|
-
|
|
3744
|
+
Minimum active-tab poll interval, ${MIN_REFRESH_SECONDS}-${MAX_REFRESH_SECONDS};
|
|
3745
|
+
safe shared grants may run later
|
|
2055
3746
|
(--refresh takes precedence)
|
|
2056
|
-
GH_GLANCE_ICONS=unicode
|
|
2057
|
-
|
|
3747
|
+
GH_GLANCE_ICONS=unicode Unicode status glyphs and text row substitutes
|
|
3748
|
+
GH_GLANCE_ICONS=ascii ASCII-only status and row icons
|
|
3749
|
+
GH_GLANCE_NO_ANIMATION=1 Stop motion; semantic status words remain
|
|
2058
3750
|
NO_COLOR=1 Disable colour (status stays readable)
|
|
2059
3751
|
INK_SCREEN_READER=true Linear, unthrottled rendering (unverified -- see README)
|
|
2060
3752
|
`;
|
|
@@ -2101,6 +3793,7 @@ if (IS_MAIN) {
|
|
|
2101
3793
|
// doctor branch on purpose, since that combination is useful and harmless.
|
|
2102
3794
|
runtime.repo = opts.repo;
|
|
2103
3795
|
runtime.host = opts.host;
|
|
3796
|
+
runtime.repoExplicit = opts.repo !== null;
|
|
2104
3797
|
runtime.verbose = opts.verbose;
|
|
2105
3798
|
if (opts.refreshMs !== null) runtime.refreshMs = opts.refreshMs;
|
|
2106
3799
|
if (opts.tabKey !== null) runtime.initialTabIndex = TAB_KEYS.indexOf(opts.tabKey);
|
|
@@ -2226,7 +3919,13 @@ const OCT_UNICODE = {
|
|
|
2226
3919
|
// blank box, which reads as "this program is broken" rather than "install a
|
|
2227
3920
|
// font", and the remedy currently lives only in --help and the README: both of
|
|
2228
3921
|
// which require quitting the full-screen app you are trying to evaluate.
|
|
2229
|
-
|
|
3922
|
+
function normalizeIconProfile(value) {
|
|
3923
|
+
if (value === "unicode" || value === "ascii") return value;
|
|
3924
|
+
return "nerd";
|
|
3925
|
+
}
|
|
3926
|
+
|
|
3927
|
+
const ICON_PROFILE = normalizeIconProfile(process.env.GH_GLANCE_ICONS);
|
|
3928
|
+
const USING_NERD_ICONS = ICON_PROFILE === "nerd";
|
|
2230
3929
|
const OCT = USING_NERD_ICONS ? OCT_NERD : OCT_UNICODE;
|
|
2231
3930
|
|
|
2232
3931
|
// ---------- Shared layout primitives ----------
|
|
@@ -3665,34 +5364,368 @@ function saveDashboardCache(path, cache, { normalized = false, base = null } = {
|
|
|
3665
5364
|
}
|
|
3666
5365
|
}
|
|
3667
5366
|
|
|
3668
|
-
function nextSecurityRaw(previousRaw, raw, blind) {
|
|
3669
|
-
if (blind === true) return null;
|
|
3670
|
-
return previousRaw === raw ? previousRaw : raw;
|
|
5367
|
+
function nextSecurityRaw(previousRaw, raw, blind) {
|
|
5368
|
+
if (blind === true) return null;
|
|
5369
|
+
return previousRaw === raw ? previousRaw : raw;
|
|
5370
|
+
}
|
|
5371
|
+
|
|
5372
|
+
const CACHE_FRESHNESS_CHECKPOINT_MS = 60_000;
|
|
5373
|
+
const SECURITY_UNCHANGED_POLL_MS = 60_000;
|
|
5374
|
+
|
|
5375
|
+
function shouldCheckpointFreshness({ persistedAt, completedAt }) {
|
|
5376
|
+
return !Number.isFinite(persistedAt) || completedAt - persistedAt >= CACHE_FRESHNESS_CHECKPOINT_MS;
|
|
5377
|
+
}
|
|
5378
|
+
|
|
5379
|
+
function securityPollDelay({ unchangedPolls, floorMs, force = false }) {
|
|
5380
|
+
if (force) return 0;
|
|
5381
|
+
return unchangedPolls > 0 ? Math.max(floorMs, SECURITY_UNCHANGED_POLL_MS) : floorMs;
|
|
5382
|
+
}
|
|
5383
|
+
|
|
5384
|
+
function shouldShowFetchLoading({ hasData, force }) {
|
|
5385
|
+
return !hasData || force === true;
|
|
5386
|
+
}
|
|
5387
|
+
|
|
5388
|
+
function tabHold(tab, heldResources = {}) {
|
|
5389
|
+
const costs = tabRequestCost(tab);
|
|
5390
|
+
const holds = RATE_RESOURCES.flatMap((resource) => {
|
|
5391
|
+
if (costs[resource] <= 0) return [];
|
|
5392
|
+
const value = heldResources[resource];
|
|
5393
|
+
if (value === true) return [{ retryAt: Number.POSITIVE_INFINITY }];
|
|
5394
|
+
return value?.held ? [value] : [];
|
|
5395
|
+
});
|
|
5396
|
+
return {
|
|
5397
|
+
held: holds.length > 0,
|
|
5398
|
+
retryAt: holds.reduce(
|
|
5399
|
+
(latest, hold) => Math.max(latest, hold.retryAt ?? Number.POSITIVE_INFINITY),
|
|
5400
|
+
0,
|
|
5401
|
+
),
|
|
5402
|
+
};
|
|
5403
|
+
}
|
|
5404
|
+
|
|
5405
|
+
function pollSchedule({
|
|
5406
|
+
nowMs,
|
|
5407
|
+
floorMs,
|
|
5408
|
+
activeKey,
|
|
5409
|
+
activeAt = nowMs,
|
|
5410
|
+
backgroundAt = nowMs + 4 * floorMs,
|
|
5411
|
+
backgroundIndex = 0,
|
|
5412
|
+
heldResources = {},
|
|
5413
|
+
}) {
|
|
5414
|
+
const due = [];
|
|
5415
|
+
let nextActiveAt = activeAt;
|
|
5416
|
+
let nextBackgroundAt = backgroundAt;
|
|
5417
|
+
let nextBackgroundIndex = backgroundIndex;
|
|
5418
|
+
if (activeAt <= nowMs) {
|
|
5419
|
+
const activeHold = tabHold(activeKey, heldResources);
|
|
5420
|
+
if (activeHold.held) nextActiveAt = activeHold.retryAt;
|
|
5421
|
+
else {
|
|
5422
|
+
due.push({ key: activeKey, kind: "active" });
|
|
5423
|
+
nextActiveAt = nowMs + floorMs;
|
|
5424
|
+
}
|
|
5425
|
+
}
|
|
5426
|
+
// Three inactive tabs share one background slot every four active periods.
|
|
5427
|
+
// Select at most one usable tab and retain the cursor for the next slot.
|
|
5428
|
+
if (backgroundAt <= nowMs) {
|
|
5429
|
+
const background = TAB_KEYS.filter((key) => key !== activeKey);
|
|
5430
|
+
let selected = -1;
|
|
5431
|
+
for (let offset = 0; offset < background.length; offset += 1) {
|
|
5432
|
+
const index = (backgroundIndex + offset) % background.length;
|
|
5433
|
+
if (!tabHold(background[index], heldResources).held) {
|
|
5434
|
+
selected = index;
|
|
5435
|
+
break;
|
|
5436
|
+
}
|
|
5437
|
+
}
|
|
5438
|
+
if (selected >= 0) {
|
|
5439
|
+
due.push({ key: background[selected], kind: "background" });
|
|
5440
|
+
nextBackgroundIndex = (selected + 1) % background.length;
|
|
5441
|
+
nextBackgroundAt = nowMs + 4 * floorMs;
|
|
5442
|
+
} else {
|
|
5443
|
+
nextBackgroundAt = background.reduce(
|
|
5444
|
+
(earliest, key) => Math.min(earliest, tabHold(key, heldResources).retryAt),
|
|
5445
|
+
Number.POSITIVE_INFINITY,
|
|
5446
|
+
);
|
|
5447
|
+
}
|
|
5448
|
+
}
|
|
5449
|
+
return {
|
|
5450
|
+
due,
|
|
5451
|
+
activeAt: nextActiveAt,
|
|
5452
|
+
backgroundAt: nextBackgroundAt,
|
|
5453
|
+
backgroundIndex: nextBackgroundIndex,
|
|
5454
|
+
nextAt: Math.min(nextActiveAt, nextBackgroundAt),
|
|
5455
|
+
};
|
|
5456
|
+
}
|
|
5457
|
+
|
|
5458
|
+
function retryPollAfterAdmissionFailure({
|
|
5459
|
+
kind,
|
|
5460
|
+
retryAt,
|
|
5461
|
+
activeAt,
|
|
5462
|
+
backgroundAt,
|
|
5463
|
+
backgroundIndex,
|
|
5464
|
+
previousBackgroundIndex,
|
|
5465
|
+
}) {
|
|
5466
|
+
if (kind === "active") {
|
|
5467
|
+
return {
|
|
5468
|
+
activeAt: Math.min(activeAt, retryAt),
|
|
5469
|
+
backgroundAt,
|
|
5470
|
+
backgroundIndex,
|
|
5471
|
+
};
|
|
5472
|
+
}
|
|
5473
|
+
if (kind === "background") {
|
|
5474
|
+
return {
|
|
5475
|
+
activeAt,
|
|
5476
|
+
backgroundAt: Math.min(backgroundAt, retryAt),
|
|
5477
|
+
backgroundIndex: previousBackgroundIndex,
|
|
5478
|
+
};
|
|
5479
|
+
}
|
|
5480
|
+
return { activeAt, backgroundAt, backgroundIndex };
|
|
5481
|
+
}
|
|
5482
|
+
|
|
5483
|
+
function governorWakeTimes(state, nowMs, floorMs, leaseId = null) {
|
|
5484
|
+
const budgets = Object.values(state?.budgets ?? {});
|
|
5485
|
+
const observedCandidates = state?.probeOutcome?.status === "failed"
|
|
5486
|
+
? []
|
|
5487
|
+
: budgets.map((budget) => budget.observedAt + BUDGET_PROBE_MS);
|
|
5488
|
+
const controlAt = Math.min(
|
|
5489
|
+
...observedCandidates,
|
|
5490
|
+
...budgets.map((budget) => budget.resetMs + BUDGET_RESET_GRACE_MS),
|
|
5491
|
+
state?.probeClaim?.leaseUntil ?? Number.POSITIVE_INFINITY,
|
|
5492
|
+
state?.probeOutcome?.nextAt ?? Number.POSITIVE_INFINITY,
|
|
5493
|
+
);
|
|
5494
|
+
const reservationAt = Object.values(state?.reservations ?? {})
|
|
5495
|
+
.filter((reservation) => reservation.status === "scheduled" &&
|
|
5496
|
+
(leaseId == null || reservation.leaseId === leaseId))
|
|
5497
|
+
.reduce((earliest, reservation) => Math.min(earliest, reservation.notBefore), Number.POSITIVE_INFINITY);
|
|
5498
|
+
return {
|
|
5499
|
+
controlAt: Number.isFinite(controlAt) ? Math.max(nowMs + 1, controlAt) : nowMs + floorMs,
|
|
5500
|
+
dataAt: Number.isFinite(reservationAt) ? Math.max(nowMs + 1, reservationAt) : Number.POSITIVE_INFINITY,
|
|
5501
|
+
};
|
|
5502
|
+
}
|
|
5503
|
+
|
|
5504
|
+
function governorDataReady(refreshResult, snapshot, activeKey, nowMs) {
|
|
5505
|
+
if (!refreshResult?.ok || !snapshot?.ok) return false;
|
|
5506
|
+
if (["waiting", "paused", "probe"].includes(refreshResult.value?.status)) return false;
|
|
5507
|
+
if (snapshot.value.probeClaim || snapshot.value.probeOutcome?.status !== "healthy") return false;
|
|
5508
|
+
const costs = tabRequestCost(activeKey);
|
|
5509
|
+
return RATE_RESOURCES.every((resource) => {
|
|
5510
|
+
if (costs[resource] <= 0) return true;
|
|
5511
|
+
const budget = snapshot.value.budgets[resource];
|
|
5512
|
+
return budget && nowMs - budget.observedAt <= BUDGET_SNAPSHOT_TTL_MS &&
|
|
5513
|
+
budget.blockUntil <= nowMs && availableForGrant({ budget, nowMs }).mode === "open";
|
|
5514
|
+
});
|
|
5515
|
+
}
|
|
5516
|
+
|
|
5517
|
+
function governorControlRetryAt(nowMs, floorMs) {
|
|
5518
|
+
return nowMs + Math.min(floorMs, 1000);
|
|
5519
|
+
}
|
|
5520
|
+
|
|
5521
|
+
function createWakeScheduler({
|
|
5522
|
+
now = Date.now,
|
|
5523
|
+
set = setTimeout,
|
|
5524
|
+
clear = clearTimeout,
|
|
5525
|
+
} = {}) {
|
|
5526
|
+
const entries = new Map();
|
|
5527
|
+
return {
|
|
5528
|
+
arm(kind, at, run) {
|
|
5529
|
+
if (!Number.isFinite(at) || at >= (entries.get(kind)?.at ?? Number.POSITIVE_INFINITY)) return false;
|
|
5530
|
+
const previous = entries.get(kind);
|
|
5531
|
+
if (previous) clear(previous.id);
|
|
5532
|
+
const entry = { at, id: null };
|
|
5533
|
+
entry.id = set(() => {
|
|
5534
|
+
if (entries.get(kind) !== entry) return;
|
|
5535
|
+
entries.delete(kind);
|
|
5536
|
+
void run();
|
|
5537
|
+
}, Math.max(0, at - now()));
|
|
5538
|
+
entries.set(kind, entry);
|
|
5539
|
+
return true;
|
|
5540
|
+
},
|
|
5541
|
+
clear(kind) {
|
|
5542
|
+
const entry = entries.get(kind);
|
|
5543
|
+
if (!entry) return false;
|
|
5544
|
+
clear(entry.id);
|
|
5545
|
+
entries.delete(kind);
|
|
5546
|
+
return true;
|
|
5547
|
+
},
|
|
5548
|
+
clearAll() {
|
|
5549
|
+
for (const entry of entries.values()) clear(entry.id);
|
|
5550
|
+
entries.clear();
|
|
5551
|
+
},
|
|
5552
|
+
at: (kind) => entries.get(kind)?.at ?? Number.POSITIVE_INFINITY,
|
|
5553
|
+
size: () => entries.size,
|
|
5554
|
+
};
|
|
5555
|
+
}
|
|
5556
|
+
|
|
5557
|
+
function createSingleFlightWake(run, onSettled = () => {}) {
|
|
5558
|
+
let running = false;
|
|
5559
|
+
let pendingArgs = null;
|
|
5560
|
+
return async (...args) => {
|
|
5561
|
+
if (running) {
|
|
5562
|
+
pendingArgs = args;
|
|
5563
|
+
return false;
|
|
5564
|
+
}
|
|
5565
|
+
running = true;
|
|
5566
|
+
try {
|
|
5567
|
+
let nextArgs = args;
|
|
5568
|
+
do {
|
|
5569
|
+
pendingArgs = null;
|
|
5570
|
+
await run(...nextArgs);
|
|
5571
|
+
nextArgs = pendingArgs;
|
|
5572
|
+
} while (nextArgs !== null);
|
|
5573
|
+
return true;
|
|
5574
|
+
} finally {
|
|
5575
|
+
running = false;
|
|
5576
|
+
onSettled();
|
|
5577
|
+
}
|
|
5578
|
+
};
|
|
5579
|
+
}
|
|
5580
|
+
|
|
5581
|
+
function pendingFailureIsTerminal(reason) {
|
|
5582
|
+
return reason === "stale";
|
|
3671
5583
|
}
|
|
3672
5584
|
|
|
3673
|
-
|
|
3674
|
-
|
|
5585
|
+
function runtimeIntentGate(liveScheduling, { force = false } = {}) {
|
|
5586
|
+
return liveScheduling
|
|
5587
|
+
? { registerIntent: true, requestProbe: false }
|
|
5588
|
+
: { registerIntent: false, requestProbe: force === true };
|
|
5589
|
+
}
|
|
3675
5590
|
|
|
3676
|
-
function
|
|
3677
|
-
|
|
5591
|
+
function admitGovernorOperation(scope, leaseId, operation, priority, nowMs, intentId = governorId()) {
|
|
5592
|
+
const costs = operationCost(operation);
|
|
5593
|
+
if (!costs || !validGovernorId(leaseId) || !validGovernorId(intentId)) {
|
|
5594
|
+
return { ok: false, reason: "corrupt" };
|
|
5595
|
+
}
|
|
5596
|
+
const decision = registerIntent(scope, {
|
|
5597
|
+
id: intentId,
|
|
5598
|
+
leaseId,
|
|
5599
|
+
tab: operation,
|
|
5600
|
+
priority,
|
|
5601
|
+
costs,
|
|
5602
|
+
requestedAt: nowMs,
|
|
5603
|
+
expiresAt: nowMs + GOVERNOR_LEASE_TTL_MS,
|
|
5604
|
+
});
|
|
5605
|
+
if (!decision.ok || decision.value.status !== "scheduled") return decision;
|
|
5606
|
+
const startAt = governorEffectiveTime(scope, nowMs);
|
|
5607
|
+
if (decision.value.notBefore > startAt) return decision;
|
|
5608
|
+
return startReservation(scope, decision.value.reservationId, startAt);
|
|
3678
5609
|
}
|
|
3679
5610
|
|
|
3680
|
-
function
|
|
3681
|
-
if (
|
|
3682
|
-
|
|
5611
|
+
function governorOutcomeForError(error) {
|
|
5612
|
+
if (error?.name === "AbortError") return "abort";
|
|
5613
|
+
if (error?.signal) return "signal";
|
|
5614
|
+
if (error?.code === "ETIMEDOUT") return "timeout";
|
|
5615
|
+
return "rejected";
|
|
3683
5616
|
}
|
|
3684
5617
|
|
|
3685
|
-
function
|
|
3686
|
-
|
|
5618
|
+
function rateLimitBlockDecision(results, resetMs) {
|
|
5619
|
+
const publications = Array.isArray(results) ? results : [];
|
|
5620
|
+
const failure = publications.find((result) => result?.ok !== true);
|
|
5621
|
+
if (publications.length === 0 || failure || !Number.isFinite(resetMs)) {
|
|
5622
|
+
return {
|
|
5623
|
+
mode: "paused",
|
|
5624
|
+
reason: failure?.reason ?? "block-unpublished",
|
|
5625
|
+
coordinationError: true,
|
|
5626
|
+
failClosed: true,
|
|
5627
|
+
};
|
|
5628
|
+
}
|
|
5629
|
+
return {
|
|
5630
|
+
mode: "paused",
|
|
5631
|
+
reason: "rate-limit",
|
|
5632
|
+
resetMs,
|
|
5633
|
+
coordinationError: false,
|
|
5634
|
+
failClosed: false,
|
|
5635
|
+
};
|
|
5636
|
+
}
|
|
5637
|
+
|
|
5638
|
+
function retryRateLimitBlockPublication(pending, nowMs, publish) {
|
|
5639
|
+
const blocks = Array.isArray(pending?.blocks) ? pending.blocks : [];
|
|
5640
|
+
if (blocks.length === 0 || blocks.some((block) =>
|
|
5641
|
+
!Number.isFinite(block.resetMs) || block.resetMs <= nowMs)) {
|
|
5642
|
+
return { status: "probe", pending };
|
|
5643
|
+
}
|
|
5644
|
+
const results = blocks.map((block) => publish(block.resource, block.resetMs));
|
|
5645
|
+
const resetMs = Math.min(...blocks.map((block) => block.resetMs));
|
|
5646
|
+
const decision = rateLimitBlockDecision(results, resetMs);
|
|
5647
|
+
return decision.failClosed
|
|
5648
|
+
? { status: "waiting", pending: { ...pending, reason: decision.reason } }
|
|
5649
|
+
: { status: "published", pending: null, decision };
|
|
5650
|
+
}
|
|
5651
|
+
|
|
5652
|
+
function mergeRateLimitBlockPublications(current, key, blocks, nowMs) {
|
|
5653
|
+
const merged = new Map(current);
|
|
5654
|
+
for (const block of Array.isArray(blocks) ? blocks : []) {
|
|
5655
|
+
if (!RATE_RESOURCES.includes(block?.resource)) continue;
|
|
5656
|
+
const previous = merged.get(block.resource);
|
|
5657
|
+
const previousBlock = previous?.blocks?.[0];
|
|
5658
|
+
merged.set(block.resource, {
|
|
5659
|
+
key,
|
|
5660
|
+
failedAt: Math.min(previous?.failedAt ?? nowMs, nowMs),
|
|
5661
|
+
reason: previous?.reason ?? "block-unpublished",
|
|
5662
|
+
blocks: [{
|
|
5663
|
+
resource: block.resource,
|
|
5664
|
+
resetMs: Number.isFinite(previousBlock?.resetMs)
|
|
5665
|
+
? previousBlock.resetMs
|
|
5666
|
+
: block.resetMs,
|
|
5667
|
+
epoch: validGovernorEpoch(previousBlock?.epoch)
|
|
5668
|
+
? previousBlock.epoch
|
|
5669
|
+
: block.epoch,
|
|
5670
|
+
}],
|
|
5671
|
+
});
|
|
5672
|
+
}
|
|
5673
|
+
return merged;
|
|
3687
5674
|
}
|
|
3688
5675
|
|
|
3689
|
-
function
|
|
5676
|
+
function hydrateRateLimitBlockPublication(pending, state) {
|
|
5677
|
+
const blocks = Array.isArray(pending?.blocks) ? pending.blocks : [];
|
|
3690
5678
|
return {
|
|
3691
|
-
|
|
3692
|
-
|
|
5679
|
+
...pending,
|
|
5680
|
+
blocks: blocks.map((block) => {
|
|
5681
|
+
const budget = state?.budgets?.[block.resource];
|
|
5682
|
+
if (!budget || budget.observedAt <= pending.failedAt) return block;
|
|
5683
|
+
return {
|
|
5684
|
+
...block,
|
|
5685
|
+
resetMs: Number.isFinite(block.resetMs) ? block.resetMs : budget.resetMs,
|
|
5686
|
+
epoch: validGovernorEpoch(block.epoch) ? block.epoch : budget.epoch,
|
|
5687
|
+
};
|
|
5688
|
+
}),
|
|
3693
5689
|
};
|
|
3694
5690
|
}
|
|
3695
5691
|
|
|
5692
|
+
function rateLimitBlockProbeRecovered(pending, state, nowMs) {
|
|
5693
|
+
const blocks = Array.isArray(pending?.blocks) ? pending.blocks : [];
|
|
5694
|
+
return blocks.length > 0 && blocks.every((block) => {
|
|
5695
|
+
const budget = state?.budgets?.[block.resource];
|
|
5696
|
+
if (!budget || budget.observedAt <= pending.failedAt) return false;
|
|
5697
|
+
if (!validGovernorEpoch(block.epoch)) return false;
|
|
5698
|
+
if (budget.epoch === block.epoch) return false;
|
|
5699
|
+
return !Number.isFinite(block.resetMs) || nowMs >= block.resetMs;
|
|
5700
|
+
});
|
|
5701
|
+
}
|
|
5702
|
+
|
|
5703
|
+
async function runAdmittedOperation({ scope, leaseId, operation, priority = "manual", signal, run }) {
|
|
5704
|
+
const admitted = admitGovernorOperation(scope, leaseId, operation, priority, Date.now());
|
|
5705
|
+
if (!admitted.ok || admitted.value.status !== "started") {
|
|
5706
|
+
if (validGovernorId(admitted.value?.intentId)) {
|
|
5707
|
+
cancelIntent(scope, admitted.value.intentId, Date.now());
|
|
5708
|
+
}
|
|
5709
|
+
const detail = admitted.value?.resetMs
|
|
5710
|
+
? ` until ${new Date(admitted.value.resetMs).toISOString()}`
|
|
5711
|
+
: admitted.value?.notBefore ? ` until ${new Date(admitted.value.notBefore).toISOString()}` : "";
|
|
5712
|
+
return { ok: false, skipped: true, decision: admitted, error: new Error(`API budget paused${detail}`) };
|
|
5713
|
+
}
|
|
5714
|
+
const reservationId = admitted.value.reservationId;
|
|
5715
|
+
try {
|
|
5716
|
+
const value = await run(signal);
|
|
5717
|
+
const costs = operationCost(operation);
|
|
5718
|
+
completeReservation(scope, reservationId, {
|
|
5719
|
+
outcome: "measured-success",
|
|
5720
|
+
actualCost: costs,
|
|
5721
|
+
}, Date.now());
|
|
5722
|
+
return { ok: true, value, reservationId };
|
|
5723
|
+
} catch (error) {
|
|
5724
|
+
completeReservation(scope, reservationId, { outcome: governorOutcomeForError(error) }, Date.now());
|
|
5725
|
+
return { ok: false, error, reservationId };
|
|
5726
|
+
}
|
|
5727
|
+
}
|
|
5728
|
+
|
|
3696
5729
|
function pollResultTransition({ key, previousRaw, raw, parse, limit, completedAt }) {
|
|
3697
5730
|
if (previousRaw === raw) return { kind: "unchanged", completedAt };
|
|
3698
5731
|
const value = parse();
|
|
@@ -3791,13 +5824,12 @@ function TabBar({ activeIndex, counts, brokenCI, firstLoad, failed, spin, useSho
|
|
|
3791
5824
|
|
|
3792
5825
|
// ---------- Status bar ----------
|
|
3793
5826
|
|
|
3794
|
-
//
|
|
3795
|
-
// Move arrows.
|
|
3796
|
-
//
|
|
3797
|
-
//
|
|
3798
|
-
// an 80-column terminal by six columns once
|
|
3799
|
-
//
|
|
3800
|
-
// over pretty-but-ambiguous. The arrow pair is a deliberate exception, and it
|
|
5827
|
+
// Control-hint glyphs are width-1 ASCII, deliberately, with one exception: the
|
|
5828
|
+
// Move arrows. Semantic status glyphs come from the selected profile below.
|
|
5829
|
+
// The return symbol and box-drawing separator stay out because they are
|
|
5830
|
+
// East-Asian-Ambiguous -- ink measures them as two columns, and a status bar
|
|
5831
|
+
// built from them overflowed an 80-column terminal by six columns once
|
|
5832
|
+
// selection added a hint. The arrow pair is a deliberate exception, and it
|
|
3801
5833
|
// is NOT covered by a width assertion -- an earlier version of this comment
|
|
3802
5834
|
// claimed `npm run test:pty` would catch a double-width rendering, and that was
|
|
3803
5835
|
// wrong twice over. Each hint is wrap: "truncate-end", so the failure mode is
|
|
@@ -3855,9 +5887,226 @@ const REMOTE_SETUP_NONINTERACTIVE_LINES = [
|
|
|
3855
5887
|
"Or use `gh-glance --repo owner/name`; Ctrl+C quits.",
|
|
3856
5888
|
];
|
|
3857
5889
|
|
|
3858
|
-
// Reserved so the hints
|
|
3859
|
-
//
|
|
3860
|
-
const
|
|
5890
|
+
// Reserved so the hints never shift when the active tab changes state. Every
|
|
5891
|
+
// status label below fits this fixed cell in both icon profiles.
|
|
5892
|
+
const REFRESH_STATUS_WIDTH = 12;
|
|
5893
|
+
|
|
5894
|
+
const REFRESH_STATUS_GLYPHS = Object.freeze({
|
|
5895
|
+
unicode: Object.freeze({
|
|
5896
|
+
setup: "·",
|
|
5897
|
+
checking: SPINNER[0],
|
|
5898
|
+
paused: "‖",
|
|
5899
|
+
waiting: "·",
|
|
5900
|
+
failed: "!",
|
|
5901
|
+
limited: "?",
|
|
5902
|
+
watching: "·",
|
|
5903
|
+
}),
|
|
5904
|
+
ascii: Object.freeze({
|
|
5905
|
+
setup: ".",
|
|
5906
|
+
checking: ".",
|
|
5907
|
+
paused: "|",
|
|
5908
|
+
waiting: ".",
|
|
5909
|
+
failed: "!",
|
|
5910
|
+
limited: "?",
|
|
5911
|
+
watching: ".",
|
|
5912
|
+
}),
|
|
5913
|
+
});
|
|
5914
|
+
|
|
5915
|
+
function isMandatoryHint(hint) {
|
|
5916
|
+
return hint?.label === "Refresh" || hint?.label === "Quit";
|
|
5917
|
+
}
|
|
5918
|
+
|
|
5919
|
+
function refreshStatus({
|
|
5920
|
+
widthMode = false,
|
|
5921
|
+
remoteSetup = false,
|
|
5922
|
+
visibleLoading = false,
|
|
5923
|
+
visibleInFlight = false,
|
|
5924
|
+
automaticStatusVisible = false,
|
|
5925
|
+
governorDecision = null,
|
|
5926
|
+
activeError = null,
|
|
5927
|
+
securityIncomplete = false,
|
|
5928
|
+
screenReader = false,
|
|
5929
|
+
} = {}) {
|
|
5930
|
+
const status = (kind, glyphKind, label, tone, animate = false, detailKind = null) => ({
|
|
5931
|
+
kind,
|
|
5932
|
+
glyphKind,
|
|
5933
|
+
label,
|
|
5934
|
+
tone,
|
|
5935
|
+
animate,
|
|
5936
|
+
detailKind,
|
|
5937
|
+
});
|
|
5938
|
+
if (widthMode) return status("width", null, "Width", "normal");
|
|
5939
|
+
if (remoteSetup) return status("setup", "setup", "Setup", "inert");
|
|
5940
|
+
if (visibleLoading) return status("checking", "checking", "Checking", "active", true);
|
|
5941
|
+
if (visibleInFlight && automaticStatusVisible && !screenReader) {
|
|
5942
|
+
return status("checking", "checking", "Checking", "active");
|
|
5943
|
+
}
|
|
5944
|
+
|
|
5945
|
+
const mode = governorDecision?.mode ?? governorDecision?.status ?? null;
|
|
5946
|
+
const detailKind = governorDecision?.detailKind ?? (
|
|
5947
|
+
Number.isFinite(governorDecision?.resetMs)
|
|
5948
|
+
? "reset"
|
|
5949
|
+
: governorDecision?.probing
|
|
5950
|
+
? "probing"
|
|
5951
|
+
: null
|
|
5952
|
+
);
|
|
5953
|
+
if (mode === "paused" || activeError?.verdict === "rate-limited") {
|
|
5954
|
+
return status("paused", "paused", "Paused", "attention", false, detailKind);
|
|
5955
|
+
}
|
|
5956
|
+
if (["waiting", "pending", "probe"].includes(mode)) {
|
|
5957
|
+
const waitingDetail = Number.isFinite(governorDecision?.notBefore)
|
|
5958
|
+
? "next"
|
|
5959
|
+
: detailKind;
|
|
5960
|
+
return status("waiting", "waiting", "Waiting", "inert", false, waitingDetail);
|
|
5961
|
+
}
|
|
5962
|
+
if (activeError) return status("failed", "failed", "Failed", "attention");
|
|
5963
|
+
if (securityIncomplete) return status("limited", "limited", "Limited", "attention");
|
|
5964
|
+
return status("watching", "watching", "Watching", "inert");
|
|
5965
|
+
}
|
|
5966
|
+
|
|
5967
|
+
function statusTime(at) {
|
|
5968
|
+
if (!Number.isFinite(at)) return null;
|
|
5969
|
+
const date = new Date(at);
|
|
5970
|
+
if (!Number.isFinite(date.getTime())) return null;
|
|
5971
|
+
return `${String(date.getHours()).padStart(2, "0")}:${String(date.getMinutes()).padStart(2, "0")}`;
|
|
5972
|
+
}
|
|
5973
|
+
|
|
5974
|
+
function statusDetailVariants(status, detail) {
|
|
5975
|
+
if (!status?.detailKind) return [];
|
|
5976
|
+
if (status.detailKind === "probing") return ["probing"];
|
|
5977
|
+
const at = status.detailKind === "next" ? detail?.notBefore : detail?.resetMs;
|
|
5978
|
+
const time = statusTime(at);
|
|
5979
|
+
if (!time) return [];
|
|
5980
|
+
return [`${status.detailKind} ${time}`, time];
|
|
5981
|
+
}
|
|
5982
|
+
|
|
5983
|
+
function statusBarLayout({
|
|
5984
|
+
cols,
|
|
5985
|
+
interactive,
|
|
5986
|
+
availableHints,
|
|
5987
|
+
status,
|
|
5988
|
+
detail = null,
|
|
5989
|
+
stale = null,
|
|
5990
|
+
version: currentVersion = "",
|
|
5991
|
+
} = {}) {
|
|
5992
|
+
const width = Number.isSafeInteger(cols) ? Math.max(0, cols) : 0;
|
|
5993
|
+
const hints = Array.isArray(availableHints) ? availableHints : [];
|
|
5994
|
+
const mandatory = hints.filter(isMandatoryHint);
|
|
5995
|
+
const optional = hints.filter((hint) => !isMandatoryHint(hint));
|
|
5996
|
+
let used = Math.min(width, REFRESH_STATUS_WIDTH);
|
|
5997
|
+
const selectedHints = [];
|
|
5998
|
+
const separatorWidth = () => selectedHints.length > 0 ? 1 : 0;
|
|
5999
|
+
const addHint = (hint, showLabel) => {
|
|
6000
|
+
const text = showLabel ? `${hint.label}: ${hint.keys}` : hint.keys;
|
|
6001
|
+
const cost = separatorWidth() + [...text].length;
|
|
6002
|
+
if (used + cost > width) return false;
|
|
6003
|
+
selectedHints.push({ ...hint, showLabel, text });
|
|
6004
|
+
used += cost;
|
|
6005
|
+
return true;
|
|
6006
|
+
};
|
|
6007
|
+
|
|
6008
|
+
for (const [index, hint] of mandatory.entries()) {
|
|
6009
|
+
// A full label is optional; every essential key is not. Reserve one
|
|
6010
|
+
// separator and the key itself for each action still to come before taking
|
|
6011
|
+
// the current action's long form. This keeps both r and q at the 24-column
|
|
6012
|
+
// terminal minimum, whose drawable frame is 23 cells wide.
|
|
6013
|
+
const remainingMinimum = mandatory.slice(index + 1).reduce(
|
|
6014
|
+
(total, candidate) => total + 1 + [...candidate.keys].length,
|
|
6015
|
+
0,
|
|
6016
|
+
);
|
|
6017
|
+
const fullText = `${hint.label}: ${hint.keys}`;
|
|
6018
|
+
const fullCost = separatorWidth() + [...fullText].length;
|
|
6019
|
+
if (used + fullCost + remainingMinimum <= width) addHint(hint, true);
|
|
6020
|
+
else addHint(hint, false);
|
|
6021
|
+
}
|
|
6022
|
+
|
|
6023
|
+
let selectedDetail = null;
|
|
6024
|
+
for (const candidate of statusDetailVariants(status, detail)) {
|
|
6025
|
+
const cost = (selectedHints.length > 0 ? 1 : 0) + candidate.length;
|
|
6026
|
+
if (used + cost <= width) {
|
|
6027
|
+
selectedDetail = candidate;
|
|
6028
|
+
used += cost;
|
|
6029
|
+
break;
|
|
6030
|
+
}
|
|
6031
|
+
}
|
|
6032
|
+
|
|
6033
|
+
const staleText = typeof stale === "string" ? stale : null;
|
|
6034
|
+
const selectedStale = staleText && used + 1 + staleText.length <= width ? staleText : null;
|
|
6035
|
+
if (selectedStale) used += 1 + selectedStale.length;
|
|
6036
|
+
|
|
6037
|
+
for (const hint of optional) {
|
|
6038
|
+
if (!addHint(hint, true)) addHint(hint, false);
|
|
6039
|
+
}
|
|
6040
|
+
|
|
6041
|
+
const versionText = String(currentVersion ?? "");
|
|
6042
|
+
const showVersion = versionText.length > 0 && used + 1 + versionText.length <= width;
|
|
6043
|
+
return {
|
|
6044
|
+
statusWidth: Math.min(width, REFRESH_STATUS_WIDTH),
|
|
6045
|
+
hints: selectedHints,
|
|
6046
|
+
mandatoryHints: selectedHints.filter(isMandatoryHint),
|
|
6047
|
+
optionalHints: selectedHints.filter((hint) => !isMandatoryHint(hint)),
|
|
6048
|
+
detail: selectedDetail,
|
|
6049
|
+
stale: selectedStale,
|
|
6050
|
+
version: showVersion ? versionText : null,
|
|
6051
|
+
interactive: Boolean(interactive),
|
|
6052
|
+
};
|
|
6053
|
+
}
|
|
6054
|
+
|
|
6055
|
+
function freshnessDeadline({
|
|
6056
|
+
lastOk,
|
|
6057
|
+
refreshMs,
|
|
6058
|
+
governorDecision = null,
|
|
6059
|
+
currentEpochs = null,
|
|
6060
|
+
} = {}) {
|
|
6061
|
+
if (!Number.isFinite(lastOk) || !Number.isFinite(refreshMs) || refreshMs < 0) return null;
|
|
6062
|
+
const baseDeadline = lastOk + Math.max(STALE_AFTER_MS, refreshMs * 6);
|
|
6063
|
+
const decisionEpochs = governorDecision?.epochs;
|
|
6064
|
+
const chargedEpochs = isRecord(decisionEpochs)
|
|
6065
|
+
? Object.entries(decisionEpochs).filter(([, epoch]) => epoch !== null)
|
|
6066
|
+
: [];
|
|
6067
|
+
const validWaitingGrant =
|
|
6068
|
+
(governorDecision?.mode ?? governorDecision?.status) === "waiting" &&
|
|
6069
|
+
Number.isFinite(governorDecision.notBefore) &&
|
|
6070
|
+
chargedEpochs.length > 0 &&
|
|
6071
|
+
isRecord(currentEpochs) &&
|
|
6072
|
+
chargedEpochs.every(([resource, epoch]) => currentEpochs[resource] === epoch);
|
|
6073
|
+
return validWaitingGrant
|
|
6074
|
+
? Math.max(baseDeadline, governorDecision.notBefore + GH_TIMEOUT_MS)
|
|
6075
|
+
: baseDeadline;
|
|
6076
|
+
}
|
|
6077
|
+
|
|
6078
|
+
function probingGovernorDecisions() {
|
|
6079
|
+
return Object.fromEntries(TABS.map((candidate) => [candidate.key, {
|
|
6080
|
+
mode: "waiting",
|
|
6081
|
+
probing: true,
|
|
6082
|
+
}]));
|
|
6083
|
+
}
|
|
6084
|
+
|
|
6085
|
+
function retainDeferredGovernorHold({ automaticStatusVisible, screenReader }) {
|
|
6086
|
+
return automaticStatusVisible === true && screenReader === true;
|
|
6087
|
+
}
|
|
6088
|
+
|
|
6089
|
+
function sameVisibleGovernorDecision(left, right) {
|
|
6090
|
+
if (left === right) return true;
|
|
6091
|
+
if (!left || !right) return false;
|
|
6092
|
+
for (const key of [
|
|
6093
|
+
"mode",
|
|
6094
|
+
"status",
|
|
6095
|
+
"reason",
|
|
6096
|
+
"resetMs",
|
|
6097
|
+
"retryAt",
|
|
6098
|
+
"notBefore",
|
|
6099
|
+
"resource",
|
|
6100
|
+
"probing",
|
|
6101
|
+
"detailKind",
|
|
6102
|
+
"coordinationError",
|
|
6103
|
+
"reservationId",
|
|
6104
|
+
]) {
|
|
6105
|
+
if (left[key] !== right[key]) return false;
|
|
6106
|
+
}
|
|
6107
|
+
return left.epochs?.core === right.epochs?.core &&
|
|
6108
|
+
left.epochs?.graphql === right.epochs?.graphql;
|
|
6109
|
+
}
|
|
3861
6110
|
|
|
3862
6111
|
function widthStatusText({ label, width, cols, saveError = false }) {
|
|
3863
6112
|
const budget = Number.isSafeInteger(cols) ? Math.max(0, cols) : 0;
|
|
@@ -3896,10 +6145,10 @@ function widthStatusText({ label, width, cols, saveError = false }) {
|
|
|
3896
6145
|
// them stay dim. The accent is the panel-title cyan rather than the amber used
|
|
3897
6146
|
// for in-progress status, so amber means exactly one thing across the product.
|
|
3898
6147
|
function StatusBar({
|
|
3899
|
-
|
|
6148
|
+
status,
|
|
6149
|
+
detail,
|
|
3900
6150
|
spin,
|
|
3901
6151
|
stale,
|
|
3902
|
-
throttle,
|
|
3903
6152
|
interactive,
|
|
3904
6153
|
cols,
|
|
3905
6154
|
remoteSetup = false,
|
|
@@ -3910,9 +6159,9 @@ function StatusBar({
|
|
|
3910
6159
|
canOpen = false,
|
|
3911
6160
|
canResize = false,
|
|
3912
6161
|
}) {
|
|
3913
|
-
// Width mode owns the whole bar, so
|
|
3914
|
-
// shown here
|
|
3915
|
-
//
|
|
6162
|
+
// Width mode owns the whole bar, so coordination detail is deliberately not
|
|
6163
|
+
// shown here. This state is transient and explicitly entered; the active-tab
|
|
6164
|
+
// refresh state returns as soon as the user leaves it.
|
|
3916
6165
|
if (widthMode && widthColumn) {
|
|
3917
6166
|
return e(
|
|
3918
6167
|
Box,
|
|
@@ -3933,58 +6182,59 @@ function StatusBar({
|
|
|
3933
6182
|
// telling the user something untrue about what the app can do. Ctrl+C still
|
|
3934
6183
|
// works there, because the tty delivers a real SIGINT.
|
|
3935
6184
|
const hints = activeKeyHints({ interactive, remoteSetup, canMove, canOpen, canResize });
|
|
3936
|
-
const
|
|
3937
|
-
|
|
3938
|
-
|
|
3939
|
-
|
|
3940
|
-
|
|
6185
|
+
const semanticStatus = status ?? refreshStatus({ remoteSetup });
|
|
6186
|
+
const layout = statusBarLayout({
|
|
6187
|
+
cols,
|
|
6188
|
+
interactive,
|
|
6189
|
+
availableHints: hints,
|
|
6190
|
+
status: semanticStatus,
|
|
6191
|
+
detail,
|
|
6192
|
+
stale,
|
|
6193
|
+
version,
|
|
6194
|
+
});
|
|
6195
|
+
const profile = ICON_PROFILE === "ascii" ? "ascii" : "unicode";
|
|
6196
|
+
const glyphs = REFRESH_STATUS_GLYPHS[profile];
|
|
6197
|
+
const glyph = profile !== "ascii" &&
|
|
6198
|
+
semanticStatus.glyphKind === "checking" && semanticStatus.animate && spin
|
|
6199
|
+
? spin
|
|
6200
|
+
: glyphs[semanticStatus.glyphKind] ?? glyphs.watching;
|
|
6201
|
+
const statusColor = semanticStatus.tone === "active"
|
|
6202
|
+
? TITLE_COLOR
|
|
6203
|
+
: semanticStatus.tone === "attention"
|
|
6204
|
+
? ATTENTION
|
|
6205
|
+
: undefined;
|
|
6206
|
+
const renderHint = (hint, index, group) => [
|
|
6207
|
+
(index > 0 || group === "optional") &&
|
|
6208
|
+
e(Text, { key: `${group}:sep:${hint.label}`, color: BORDER_COLOR }, " "),
|
|
6209
|
+
e(
|
|
6210
|
+
Text,
|
|
6211
|
+
{ key: `${group}:${hint.label}`, wrap: "truncate-end" },
|
|
6212
|
+
hint.showLabel ? e(Text, { dimColor: true }, `${hint.label}: `) : null,
|
|
6213
|
+
e(Text, { color: TITLE_COLOR, bold: true }, hint.keys),
|
|
6214
|
+
),
|
|
6215
|
+
].filter(Boolean);
|
|
3941
6216
|
return e(
|
|
3942
6217
|
Box,
|
|
3943
6218
|
{ flexDirection: "row" },
|
|
3944
6219
|
e(
|
|
3945
6220
|
Box,
|
|
3946
|
-
{ width:
|
|
3947
|
-
// Pinned to the resting frame when idle: the counter also drives the run
|
|
3948
|
-
// icons, so borrowing it here would set this turning while a workflow
|
|
3949
|
-
// executes -- saying "fetching" at a moment when nothing is being
|
|
3950
|
-
// fetched.
|
|
6221
|
+
{ width: layout.statusWidth, flexShrink: 0 },
|
|
3951
6222
|
e(
|
|
3952
6223
|
Text,
|
|
3953
|
-
{ color:
|
|
3954
|
-
|
|
6224
|
+
{ color: statusColor, dimColor: semanticStatus.tone === "inert", wrap: "truncate-end" },
|
|
6225
|
+
`${glyph} ${semanticStatus.label}`,
|
|
3955
6226
|
),
|
|
3956
6227
|
),
|
|
3957
|
-
|
|
3958
|
-
|
|
3959
|
-
|
|
3960
|
-
// is worth getting the column back for the other 99% of the time.
|
|
3961
|
-
stale
|
|
3962
|
-
? e(Box, { marginRight: 1, flexShrink: 0 }, e(Text, { color: ATTENTION }, stale))
|
|
6228
|
+
...layout.mandatoryHints.flatMap((hint, index) => renderHint(hint, index, "mandatory")),
|
|
6229
|
+
layout.detail
|
|
6230
|
+
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { dimColor: true }, layout.detail))
|
|
3963
6231
|
: null,
|
|
3964
|
-
|
|
3965
|
-
|
|
3966
|
-
// like a problem would send the user looking for one.
|
|
3967
|
-
throttle
|
|
3968
|
-
? e(Box, { marginRight: 1, flexShrink: 0 }, e(Text, { dimColor: true }, throttle))
|
|
6232
|
+
layout.stale
|
|
6233
|
+
? e(Box, { marginLeft: 1, flexShrink: 0 }, e(Text, { color: ATTENTION }, layout.stale))
|
|
3969
6234
|
: null,
|
|
3970
|
-
...
|
|
3971
|
-
|
|
3972
|
-
|
|
3973
|
-
e(Text, { key: `sep${i}`, color: BORDER_COLOR }, compact ? " " : " | "),
|
|
3974
|
-
e(
|
|
3975
|
-
Text,
|
|
3976
|
-
{ key: hint.label, wrap: "truncate-end" },
|
|
3977
|
-
compact ? null : e(Text, { dimColor: true }, `${hint.label}: `),
|
|
3978
|
-
e(Text, { color: TITLE_COLOR, bold: true }, hint.keys),
|
|
3979
|
-
),
|
|
3980
|
-
])
|
|
3981
|
-
.filter(Boolean),
|
|
3982
|
-
// Right-aligned, like lazygit's footer. Dropped in compact mode rather than
|
|
3983
|
-
// left to shrink alongside the hints: it would compete for the same shrink
|
|
3984
|
-
// budget as "Quit", the one hint the comment above already fought to keep
|
|
3985
|
-
// on screen down to 45 columns, and version digits are not worth that.
|
|
3986
|
-
!compact && e(Box, { flexGrow: 1 }),
|
|
3987
|
-
!compact && e(Text, { key: "version", dimColor: true, wrap: "truncate-end" }, version),
|
|
6235
|
+
...layout.optionalHints.flatMap((hint, index) => renderHint(hint, index, "optional")),
|
|
6236
|
+
layout.version && e(Box, { flexGrow: 1 }),
|
|
6237
|
+
layout.version && e(Text, { key: "version", dimColor: true, wrap: "truncate-end" }, layout.version),
|
|
3988
6238
|
);
|
|
3989
6239
|
}
|
|
3990
6240
|
|
|
@@ -4034,6 +6284,7 @@ function useTerminalSize(stdout) {
|
|
|
4034
6284
|
// ---------- App ----------
|
|
4035
6285
|
|
|
4036
6286
|
function App({ onCreateRemote = () => {} } = {}) {
|
|
6287
|
+
const screenReader = process.env.INK_SCREEN_READER === "true";
|
|
4037
6288
|
const { stdout } = useStdout();
|
|
4038
6289
|
const { isRawModeSupported } = useStdin();
|
|
4039
6290
|
const { exit, suspendTerminal } = useApp();
|
|
@@ -4141,10 +6392,13 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4141
6392
|
const [iconHintDue, setIconHintDue] = useState(false);
|
|
4142
6393
|
const [errors, setErrors] = useState({ actions: null, issues: null, prs: null, security: null });
|
|
4143
6394
|
const [failureContext, setFailureContext] = useState(null);
|
|
4144
|
-
const [loading, setLoading] = useState({ actions:
|
|
4145
|
-
|
|
4146
|
-
|
|
4147
|
-
const [
|
|
6395
|
+
const [loading, setLoading] = useState({ actions: false, issues: false, prs: false, security: false });
|
|
6396
|
+
const [waiting, setWaiting] = useState({ actions: true, issues: true, prs: true, security: true });
|
|
6397
|
+
const [governorDecisions, setGovernorDecisions] = useState(probingGovernorDecisions);
|
|
6398
|
+
const [governorEpochs, setGovernorEpochs] = useState(null);
|
|
6399
|
+
const [requestStatuses, setRequestStatuses] = useState(() =>
|
|
6400
|
+
Object.fromEntries(TABS.map((candidate) => [candidate.key, null])),
|
|
6401
|
+
);
|
|
4148
6402
|
const [now, setNow] = useState(new Date());
|
|
4149
6403
|
const { rows, cols, useShortLabels } = useTerminalSize(stdout);
|
|
4150
6404
|
const [frame, setFrame] = useState(0);
|
|
@@ -4158,6 +6412,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4158
6412
|
|
|
4159
6413
|
const tab = TABS[activeIndex];
|
|
4160
6414
|
const tabError = errors[tab.key];
|
|
6415
|
+
const activeGovernorDecision = governorDecisions[tab.key];
|
|
6416
|
+
const activeRequestStatus = requestStatuses[tab.key];
|
|
4161
6417
|
// Once any endpoint proves the folder has no remote, the whole dashboard is
|
|
4162
6418
|
// in setup mode. Keeping this tab-local made a quick tab switch replace the
|
|
4163
6419
|
// onboarding prompt with a second raw fetch failure while another command
|
|
@@ -4188,6 +6444,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4188
6444
|
// wrong by one row is what makes ink repaint the whole frame.
|
|
4189
6445
|
const extraLines =
|
|
4190
6446
|
(tabError && !remoteSetup ? 1 : 0) +
|
|
6447
|
+
(activeGovernorDecision?.coordinationError && !remoteSetup ? 1 : 0) +
|
|
4191
6448
|
(tab.key === "security" && !remoteSetup ? securityLines.length : 0);
|
|
4192
6449
|
// Reserve lines for: the tab bar and the divider under it (2), the panel's
|
|
4193
6450
|
// top and bottom edges (2), the column header and its separator (2), and
|
|
@@ -4233,6 +6490,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4233
6490
|
);
|
|
4234
6491
|
const fetchTabRef = useRef(null);
|
|
4235
6492
|
const contextCoordinatorRef = useRef(null);
|
|
6493
|
+
const governorRef = useRef(null);
|
|
4236
6494
|
|
|
4237
6495
|
const interactive = Boolean(isRawModeSupported);
|
|
4238
6496
|
|
|
@@ -4502,7 +6760,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4502
6760
|
// failure surfaces through the tab's normal error line rather than as an
|
|
4503
6761
|
// unhandled rejection.
|
|
4504
6762
|
openRequests
|
|
4505
|
-
.start(guard, ({ signal }) => openInBrowser(tabKey, item, signal))
|
|
6763
|
+
.start(guard, ({ signal }) => openInBrowser(tabKey, item, signal, governorRef.current))
|
|
4506
6764
|
?.catch((err) => {
|
|
4507
6765
|
if (err?.name === "AbortError") return;
|
|
4508
6766
|
setErrors((x) => ({ ...x, [tabKey]: textTabError(err) }));
|
|
@@ -4573,7 +6831,6 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4573
6831
|
// bypasses (and clears) any failure backoff: this key is the user saying
|
|
4574
6832
|
// "try again now", and a refresh that silently declined to refresh would
|
|
4575
6833
|
// be worse than no key at all.
|
|
4576
|
-
contextCoordinatorRef.current?.invalidate();
|
|
4577
6834
|
fetchTabRef.current?.(TABS[activeIndexRef.current].key, { force: true });
|
|
4578
6835
|
} else if (input >= "1" && input <= String(TABS.length)) {
|
|
4579
6836
|
setActiveIndex(Number(input) - 1);
|
|
@@ -4592,28 +6849,19 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4592
6849
|
{ isActive: interactive },
|
|
4593
6850
|
);
|
|
4594
6851
|
|
|
4595
|
-
// Deliberately mount-only. Every value this closure needs is read
|
|
4596
|
-
// ref
|
|
4597
|
-
//
|
|
4598
|
-
// `data` to the dependency array would tear down and rebuild the poll loop on
|
|
4599
|
-
// every tab keypress and every terminal resize, cancelling in-flight requests
|
|
4600
|
-
// -- the behaviour the per-tab guard and the fetch-time limit read were
|
|
4601
|
-
// written to prevent. Do not "fix" this by adding them.
|
|
6852
|
+
// Deliberately mount-only. Every changing value this closure needs is read
|
|
6853
|
+
// through a ref so tab changes and terminal resizes do not tear down the
|
|
6854
|
+
// lease, timers, or in-flight work.
|
|
4602
6855
|
//
|
|
4603
6856
|
// exhaustive-deps is satisfied as written -- every captured value is a ref,
|
|
4604
6857
|
// a setState function, or the stable cache writer created once above. If a
|
|
4605
6858
|
// changing dependency ever becomes necessary, that is the signal that the
|
|
4606
6859
|
// mount-only contract has been broken.
|
|
4607
6860
|
//
|
|
4608
|
-
// The
|
|
4609
|
-
//
|
|
4610
|
-
// still runs exactly once and no closure, ref or AbortController is ever
|
|
4611
|
-
// recreated. What is created once is the effect; what changes is only the
|
|
4612
|
-
// delay `tick` is scheduled at. Do not move the re-arm into a dependency
|
|
4613
|
-
// array -- that is precisely the teardown this comment exists to prevent.
|
|
6861
|
+
// The data, control, and heartbeat schedulers are independent one-shot
|
|
6862
|
+
// timers. Each callback computes and arms its next wake inside this effect.
|
|
4614
6863
|
useEffect(() => {
|
|
4615
6864
|
let cancelled = false;
|
|
4616
|
-
let ticks = 0;
|
|
4617
6865
|
const controller = new AbortController();
|
|
4618
6866
|
registerLiveAbort(controller);
|
|
4619
6867
|
const withTargetHost = (context) => ({
|
|
@@ -4626,7 +6874,7 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4626
6874
|
}),
|
|
4627
6875
|
});
|
|
4628
6876
|
const coordinator = createFailureContextCoordinator({
|
|
4629
|
-
resolve: async (signal) => withTargetHost(await resolveFailureContext(signal)),
|
|
6877
|
+
resolve: async (signal) => withTargetHost(await resolveFailureContext(signal, governorRef.current)),
|
|
4630
6878
|
commit: (context) => {
|
|
4631
6879
|
if (!cancelled) setFailureContext(context);
|
|
4632
6880
|
},
|
|
@@ -4667,27 +6915,165 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4667
6915
|
let securityUnchangedPolls = 0;
|
|
4668
6916
|
let securityNextPollAt = 0;
|
|
4669
6917
|
|
|
4670
|
-
function
|
|
6918
|
+
function setTabGovernorDecision(key, decision) {
|
|
6919
|
+
setGovernorDecisions((current) => sameVisibleGovernorDecision(current[key], decision)
|
|
6920
|
+
? current
|
|
6921
|
+
: { ...current, [key]: decision });
|
|
6922
|
+
}
|
|
6923
|
+
|
|
6924
|
+
function setTabWaiting(key, value) {
|
|
6925
|
+
setWaiting((current) => current[key] === value
|
|
6926
|
+
? current
|
|
6927
|
+
: { ...current, [key]: value });
|
|
6928
|
+
}
|
|
6929
|
+
|
|
6930
|
+
function publishGovernorEpochs(snapshot) {
|
|
6931
|
+
const epochs = snapshot?.value?.epochs;
|
|
6932
|
+
if (!isRecord(epochs)) return;
|
|
6933
|
+
setGovernorEpochs((current) =>
|
|
6934
|
+
current?.core === epochs.core && current?.graphql === epochs.graphql
|
|
6935
|
+
? current
|
|
6936
|
+
: { core: epochs.core ?? null, graphql: epochs.graphql ?? null });
|
|
6937
|
+
}
|
|
6938
|
+
|
|
6939
|
+
function visibleGovernorDecision(decision) {
|
|
6940
|
+
const mode = decision?.mode ?? decision?.status ?? "pending";
|
|
6941
|
+
if (mode === "paused" || decision?.reason === "reset") {
|
|
6942
|
+
return {
|
|
6943
|
+
...decision,
|
|
6944
|
+
mode: "paused",
|
|
6945
|
+
detailKind: Number.isFinite(decision?.resetMs) ? "reset" : decision?.detailKind,
|
|
6946
|
+
};
|
|
6947
|
+
}
|
|
6948
|
+
if (["waiting", "pending", "probe"].includes(mode) ||
|
|
6949
|
+
mode === "scheduled" && Number.isFinite(decision?.notBefore)) {
|
|
6950
|
+
return {
|
|
6951
|
+
...decision,
|
|
6952
|
+
mode: "waiting",
|
|
6953
|
+
probing: mode === "probe" || decision?.reason === "budget-unknown",
|
|
6954
|
+
};
|
|
6955
|
+
}
|
|
6956
|
+
return null;
|
|
6957
|
+
}
|
|
6958
|
+
|
|
6959
|
+
function pauseCoordination(key, reason) {
|
|
6960
|
+
setTabGovernorDecision(key, {
|
|
6961
|
+
mode: "paused",
|
|
6962
|
+
reason: reason ?? "unavailable",
|
|
6963
|
+
coordinationError: true,
|
|
6964
|
+
});
|
|
6965
|
+
}
|
|
6966
|
+
|
|
6967
|
+
function publishControlStatus(key, refreshed, snapshot, nowMs) {
|
|
6968
|
+
if (pendingBlockPublications.size > 0) {
|
|
6969
|
+
pauseCoordination(key, "block-unpublished");
|
|
6970
|
+
return;
|
|
6971
|
+
}
|
|
6972
|
+
if (!refreshed?.ok) {
|
|
6973
|
+
pauseCoordination(key, refreshed?.reason);
|
|
6974
|
+
return;
|
|
6975
|
+
}
|
|
6976
|
+
if (!snapshot?.ok) {
|
|
6977
|
+
pauseCoordination(key, snapshot?.reason);
|
|
6978
|
+
return;
|
|
6979
|
+
}
|
|
6980
|
+
publishGovernorEpochs(snapshot);
|
|
6981
|
+
if (refreshed.value?.status === "waiting") {
|
|
6982
|
+
setTabGovernorDecision(key, { mode: "waiting", probing: true });
|
|
6983
|
+
return;
|
|
6984
|
+
}
|
|
6985
|
+
if (screenReader && inFlightRef.current[key]) return;
|
|
6986
|
+
const pendingIntent = pending.get(key);
|
|
6987
|
+
const reservation = pendingIntent
|
|
6988
|
+
? snapshot.value.reservations[`reservation:${pendingIntent.intentId}`]
|
|
6989
|
+
: null;
|
|
6990
|
+
if (reservation?.status === "scheduled" && reservation.notBefore > nowMs) {
|
|
6991
|
+
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
6992
|
+
status: "scheduled",
|
|
6993
|
+
reservationId: `reservation:${pendingIntent.intentId}`,
|
|
6994
|
+
...reservation,
|
|
6995
|
+
}));
|
|
6996
|
+
return;
|
|
6997
|
+
}
|
|
6998
|
+
const costs = tabRequestCost(key);
|
|
6999
|
+
for (const resource of RATE_RESOURCES) {
|
|
7000
|
+
if (costs[resource] <= 0) continue;
|
|
7001
|
+
const budget = snapshot.value.budgets[resource];
|
|
7002
|
+
const decision = budget
|
|
7003
|
+
? resourceDecision({
|
|
7004
|
+
budget,
|
|
7005
|
+
resource,
|
|
7006
|
+
leases: snapshot.value.leases,
|
|
7007
|
+
reservations: Object.values(snapshot.value.reservations),
|
|
7008
|
+
nowMs,
|
|
7009
|
+
cost: costs[resource],
|
|
7010
|
+
})
|
|
7011
|
+
: { mode: "probe", reason: "budget-unknown" };
|
|
7012
|
+
if (decision.mode === "open") continue;
|
|
7013
|
+
const resetHold = ["budget-reset", "reset", "rate-limit"].includes(decision.reason) ||
|
|
7014
|
+
budget?.blockUntil > nowMs;
|
|
7015
|
+
setTabGovernorDecision(key, {
|
|
7016
|
+
mode: resetHold || decision.mode === "paused" ? "paused" : "waiting",
|
|
7017
|
+
reason: decision.reason,
|
|
7018
|
+
resetMs: budget?.resetMs ?? decision.resetMs ?? null,
|
|
7019
|
+
probing: decision.mode === "probe" && !resetHold,
|
|
7020
|
+
});
|
|
7021
|
+
return;
|
|
7022
|
+
}
|
|
7023
|
+
setTabGovernorDecision(key, null);
|
|
7024
|
+
}
|
|
7025
|
+
|
|
7026
|
+
function commit(key, run, {
|
|
7027
|
+
force = false,
|
|
7028
|
+
scope,
|
|
7029
|
+
reservationId,
|
|
7030
|
+
onSettled,
|
|
7031
|
+
automaticStatusVisible = false,
|
|
7032
|
+
} = {}) {
|
|
4671
7033
|
// Per-tab rather than one flag for the whole tick, so switching tabs can
|
|
4672
7034
|
// refresh the tab you just landed on without waiting on an unrelated
|
|
4673
7035
|
// background fetch -- and so a slow repo can't stack refreshes.
|
|
4674
7036
|
if (inFlightRef.current[key]) return Promise.resolve();
|
|
4675
7037
|
inFlightRef.current[key] = true;
|
|
4676
|
-
|
|
4677
|
-
for (const backoffKey of forcedBackoffKeys(key)) clearBackoff(backoffKey);
|
|
4678
|
-
}
|
|
7038
|
+
clearForcedBackoffAfterStart(key, force, "started");
|
|
4679
7039
|
const visibleLoading = shouldShowFetchLoading({
|
|
4680
7040
|
hasData: dataRef.current[key] !== null,
|
|
4681
7041
|
force,
|
|
4682
7042
|
});
|
|
4683
7043
|
if (visibleLoading) setLoading((l) => (l[key] ? l : { ...l, [key]: true }));
|
|
7044
|
+
const retainDeferredHold = retainDeferredGovernorHold({
|
|
7045
|
+
automaticStatusVisible,
|
|
7046
|
+
screenReader,
|
|
7047
|
+
});
|
|
7048
|
+
if (!retainDeferredHold) setTabGovernorDecision(key, null);
|
|
7049
|
+
const publishRequestStatus = visibleLoading ||
|
|
7050
|
+
(automaticStatusVisible && !screenReader);
|
|
7051
|
+
if (publishRequestStatus) {
|
|
7052
|
+
setRequestStatuses((current) => ({
|
|
7053
|
+
...current,
|
|
7054
|
+
[key]: { automaticStatusVisible },
|
|
7055
|
+
}));
|
|
7056
|
+
}
|
|
4684
7057
|
return run()
|
|
4685
7058
|
.then((result) => {
|
|
4686
|
-
|
|
4687
|
-
|
|
4688
|
-
|
|
4689
|
-
|
|
4690
|
-
|
|
7059
|
+
completeReservation(scope, reservationId, result?.measuredSuccess === false
|
|
7060
|
+
? { outcome: "rejected" }
|
|
7061
|
+
: {
|
|
7062
|
+
outcome: "measured-success",
|
|
7063
|
+
actualCost: {
|
|
7064
|
+
core: result?.restSpent ?? REST_PER_FETCH[key] ?? 0,
|
|
7065
|
+
graphql: result?.graphqlSpent ?? GRAPHQL_PER_FETCH[key] ?? 0,
|
|
7066
|
+
},
|
|
7067
|
+
}, Date.now());
|
|
7068
|
+
if (result?.rateLimited) {
|
|
7069
|
+
const blockAt = Date.now();
|
|
7070
|
+
const budget = inspectGovernor(scope, blockAt).value?.budgets?.core;
|
|
7071
|
+
handleRateLimitBlocks(key, [{
|
|
7072
|
+
resource: "core",
|
|
7073
|
+
resetMs: Number.isFinite(budget?.resetMs) ? budget.resetMs : null,
|
|
7074
|
+
epoch: budget?.epoch ?? null,
|
|
7075
|
+
}], blockAt);
|
|
7076
|
+
}
|
|
4691
7077
|
if (cancelled) return;
|
|
4692
7078
|
const { raw, parse, limit } = result;
|
|
4693
7079
|
// Identical payload: skip the parse *and* the state update. Returning
|
|
@@ -4784,12 +7170,9 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4784
7170
|
});
|
|
4785
7171
|
})
|
|
4786
7172
|
.catch((err) => {
|
|
4787
|
-
|
|
4788
|
-
|
|
4789
|
-
|
|
4790
|
-
// errs toward throttling harder, which is the safe direction.
|
|
4791
|
-
spentTotal.core += REST_PER_FETCH[key] ?? 0;
|
|
4792
|
-
spentTotal.graphql += GRAPHQL_PER_FETCH[key] ?? 0;
|
|
7173
|
+
completeReservation(scope, reservationId, {
|
|
7174
|
+
outcome: governorOutcomeForError(err),
|
|
7175
|
+
}, Date.now());
|
|
4793
7176
|
if (cancelled || err?.name === "AbortError") return;
|
|
4794
7177
|
// Preserve both the verdict and the bounded raw error in state. The
|
|
4795
7178
|
// renderer translates recognized verdicts at draw time, which lets a
|
|
@@ -4804,133 +7187,557 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4804
7187
|
// subprocesses an hour, indefinitely, against a token that is already
|
|
4805
7188
|
// refusing. "other" has no ladder on purpose: a network drop should
|
|
4806
7189
|
// recover on the very next tick once the network is back.
|
|
4807
|
-
const steps = pick(FAILURE_LADDER, verdict, null);
|
|
7190
|
+
const steps = verdict === "rate-limited" ? null : pick(FAILURE_LADDER, verdict, null);
|
|
4808
7191
|
if (steps) recordFailure(`tab:${key}`, performance.now(), steps);
|
|
7192
|
+
if (verdict === "rate-limited") {
|
|
7193
|
+
const snapshot = inspectGovernor(scope, Date.now());
|
|
7194
|
+
const costs = tabRequestCost(key);
|
|
7195
|
+
const blocks = [];
|
|
7196
|
+
for (const resource of RATE_RESOURCES) {
|
|
7197
|
+
if (costs[resource] <= 0) continue;
|
|
7198
|
+
const resourceResetMs = snapshot.value?.budgets?.[resource]?.resetMs;
|
|
7199
|
+
blocks.push({
|
|
7200
|
+
resource,
|
|
7201
|
+
resetMs: Number.isFinite(resourceResetMs) ? resourceResetMs : null,
|
|
7202
|
+
epoch: snapshot.value?.budgets?.[resource]?.epoch ?? null,
|
|
7203
|
+
});
|
|
7204
|
+
}
|
|
7205
|
+
handleRateLimitBlocks(key, blocks, Date.now());
|
|
7206
|
+
}
|
|
4809
7207
|
})
|
|
4810
7208
|
.finally(() => {
|
|
4811
7209
|
inFlightRef.current[key] = false;
|
|
7210
|
+
onSettled?.();
|
|
4812
7211
|
if (!cancelled) {
|
|
4813
7212
|
setLoading((l) => (l[key] ? { ...l, [key]: false } : l));
|
|
7213
|
+
if (publishRequestStatus) {
|
|
7214
|
+
setRequestStatuses((current) => current[key] === null
|
|
7215
|
+
? current
|
|
7216
|
+
: { ...current, [key]: null });
|
|
7217
|
+
}
|
|
7218
|
+
if (retainDeferredHold) {
|
|
7219
|
+
setGovernorDecisions((current) =>
|
|
7220
|
+
current[key]?.reservationId === reservationId
|
|
7221
|
+
? { ...current, [key]: null }
|
|
7222
|
+
: current);
|
|
7223
|
+
}
|
|
4814
7224
|
}
|
|
4815
7225
|
});
|
|
4816
7226
|
}
|
|
4817
7227
|
|
|
4818
|
-
|
|
4819
|
-
|
|
4820
|
-
|
|
4821
|
-
|
|
4822
|
-
|
|
7228
|
+
const leaseId = governorId();
|
|
7229
|
+
const pending = new Map();
|
|
7230
|
+
const wakeScheduler = createWakeScheduler();
|
|
7231
|
+
let scope = null;
|
|
7232
|
+
let cleanupScope = null;
|
|
7233
|
+
let registeredScopeHash = null;
|
|
7234
|
+
let remoteUrls = [];
|
|
7235
|
+
let liveScheduling = false;
|
|
7236
|
+
let activePollAt = Number.POSITIVE_INFINITY;
|
|
7237
|
+
let backgroundPollAt = Number.POSITIVE_INFINITY;
|
|
7238
|
+
let backgroundIndex = 0;
|
|
7239
|
+
let pendingBlockPublications = new Map();
|
|
7240
|
+
|
|
7241
|
+
function armWake(kind, at, run) {
|
|
7242
|
+
if (!cancelled) wakeScheduler.arm(kind, at, run);
|
|
7243
|
+
}
|
|
7244
|
+
|
|
7245
|
+
function failClosedRateLimit(key, reason) {
|
|
7246
|
+
liveScheduling = false;
|
|
7247
|
+
activePollAt = Number.POSITIVE_INFINITY;
|
|
7248
|
+
backgroundPollAt = Number.POSITIVE_INFINITY;
|
|
7249
|
+
wakeScheduler.clear("data");
|
|
7250
|
+
pauseCoordination(key, reason);
|
|
7251
|
+
armWake("control", governorControlRetryAt(Date.now(), runtime.refreshMs), controlWake);
|
|
7252
|
+
}
|
|
7253
|
+
|
|
7254
|
+
function attemptPendingBlockPublications(currentScope, nowMs, state = null, requestProbe = false) {
|
|
7255
|
+
for (const [resource, original] of [...pendingBlockPublications]) {
|
|
7256
|
+
if (state && rateLimitBlockProbeRecovered(original, state, nowMs)) {
|
|
7257
|
+
pendingBlockPublications.delete(resource);
|
|
7258
|
+
continue;
|
|
7259
|
+
}
|
|
7260
|
+
const candidate = state
|
|
7261
|
+
? hydrateRateLimitBlockPublication(original, state)
|
|
7262
|
+
: original;
|
|
7263
|
+
const attempt = retryRateLimitBlockPublication(
|
|
7264
|
+
candidate,
|
|
7265
|
+
nowMs,
|
|
7266
|
+
(name, resetMs) => recordResourceBlock(currentScope, name, resetMs, "rate-limit"),
|
|
7267
|
+
);
|
|
7268
|
+
if (attempt.status === "published") {
|
|
7269
|
+
pendingBlockPublications.delete(resource);
|
|
7270
|
+
setTabGovernorDecision(candidate.key, attempt.decision);
|
|
7271
|
+
continue;
|
|
7272
|
+
}
|
|
7273
|
+
pendingBlockPublications.set(resource, attempt.pending);
|
|
7274
|
+
if (attempt.status === "probe" && requestProbe) {
|
|
7275
|
+
const budget = state?.budgets?.[resource];
|
|
7276
|
+
if (budget) requestManualProbe(
|
|
7277
|
+
currentScope,
|
|
7278
|
+
leaseId,
|
|
7279
|
+
budget.epoch,
|
|
7280
|
+
budget.observedAt,
|
|
7281
|
+
nowMs,
|
|
7282
|
+
);
|
|
7283
|
+
}
|
|
7284
|
+
}
|
|
7285
|
+
}
|
|
7286
|
+
|
|
7287
|
+
function handleRateLimitBlocks(key, blocks, nowMs) {
|
|
7288
|
+
pendingBlockPublications = mergeRateLimitBlockPublications(
|
|
7289
|
+
pendingBlockPublications,
|
|
7290
|
+
key,
|
|
7291
|
+
blocks,
|
|
7292
|
+
nowMs,
|
|
7293
|
+
);
|
|
7294
|
+
attemptPendingBlockPublications(scope, nowMs);
|
|
7295
|
+
if (pendingBlockPublications.size > 0) {
|
|
7296
|
+
failClosedRateLimit(key, pendingBlockPublications.values().next().value?.reason);
|
|
7297
|
+
}
|
|
7298
|
+
}
|
|
7299
|
+
|
|
7300
|
+
function identity() {
|
|
7301
|
+
return {
|
|
7302
|
+
effectiveHost: effectiveRuntimeHost({ remoteUrls }),
|
|
7303
|
+
authIdentity: authCacheIdentity(),
|
|
7304
|
+
};
|
|
7305
|
+
}
|
|
7306
|
+
|
|
7307
|
+
function resetVisibleScope(nowMs, { immediate = false } = {}) {
|
|
7308
|
+
pending.clear();
|
|
7309
|
+
pendingBlockPublications.clear();
|
|
7310
|
+
liveScheduling = false;
|
|
7311
|
+
activePollAt = Number.POSITIVE_INFINITY;
|
|
7312
|
+
backgroundPollAt = Number.POSITIVE_INFINITY;
|
|
7313
|
+
wakeScheduler.clear("data");
|
|
7314
|
+
wakeScheduler.clear("heartbeat");
|
|
7315
|
+
wakeScheduler.clear("control");
|
|
7316
|
+
setGovernorEpochs(null);
|
|
7317
|
+
setGovernorDecisions(probingGovernorDecisions());
|
|
7318
|
+
armWake(
|
|
7319
|
+
"control",
|
|
7320
|
+
immediate ? nowMs + 1 : governorControlRetryAt(nowMs, runtime.refreshMs),
|
|
7321
|
+
controlWake,
|
|
7322
|
+
);
|
|
7323
|
+
}
|
|
7324
|
+
|
|
7325
|
+
function ensureScope(nowMs = Date.now(), { maintain = false } = {}) {
|
|
7326
|
+
const current = identity();
|
|
7327
|
+
const nextHash = governorScopeHash(current.effectiveHost, current.authIdentity);
|
|
7328
|
+
if (!nextHash) {
|
|
7329
|
+
if (scope !== null) {
|
|
7330
|
+
if (cleanupScope && registeredScopeHash) releaseLease(cleanupScope, leaseId);
|
|
7331
|
+
registeredScopeHash = null;
|
|
7332
|
+
if (governorRef.current?.leaseId === leaseId) governorRef.current = null;
|
|
7333
|
+
scope = null;
|
|
7334
|
+
cleanupScope = null;
|
|
7335
|
+
resetVisibleScope(nowMs);
|
|
7336
|
+
}
|
|
7337
|
+
return null;
|
|
7338
|
+
}
|
|
7339
|
+
const migrating = scope !== null && scope.hash !== nextHash;
|
|
7340
|
+
if (scope?.hash === nextHash && registeredScopeHash === nextHash) {
|
|
7341
|
+
if (!maintain) return scope;
|
|
7342
|
+
const activeTab = TABS[activeIndexRef.current].key;
|
|
7343
|
+
const kept = maintainControlLease(scope, leaseId, runtime.refreshMs, activeTab, nowMs);
|
|
7344
|
+
if (kept.ok) return scope;
|
|
7345
|
+
registeredScopeHash = null;
|
|
7346
|
+
if (governorRef.current?.leaseId === leaseId) governorRef.current = null;
|
|
7347
|
+
return null;
|
|
7348
|
+
}
|
|
7349
|
+
if (scope?.hash !== nextHash) {
|
|
7350
|
+
if (cleanupScope && registeredScopeHash) releaseLease(cleanupScope, leaseId);
|
|
7351
|
+
registeredScopeHash = null;
|
|
7352
|
+
if (migrating) {
|
|
7353
|
+
// The failed publication belonged to the old account scope. Releasing
|
|
7354
|
+
// that lease guarantees this process will start no more work there;
|
|
7355
|
+
// the new scope must establish its own authoritative budget instead
|
|
7356
|
+
// of inheriting an unrelated account's local hold.
|
|
7357
|
+
resetVisibleScope(nowMs, { immediate: true });
|
|
7358
|
+
}
|
|
7359
|
+
const created = createGovernorScope({ ...current, identityProvider: identity });
|
|
7360
|
+
if (!created.ok) return null;
|
|
7361
|
+
scope = created.value;
|
|
7362
|
+
cleanupScope = { ...scope, identityProvider: null };
|
|
7363
|
+
}
|
|
7364
|
+
const activeTab = TABS[activeIndexRef.current].key;
|
|
7365
|
+
const registered = registerLease(scope, {
|
|
7366
|
+
id: leaseId,
|
|
7367
|
+
expiresAt: nowMs + GOVERNOR_LEASE_TTL_MS,
|
|
7368
|
+
floorMs: runtime.refreshMs,
|
|
7369
|
+
activeTab,
|
|
7370
|
+
phaseSeed: { seed: leaseId, registeredAt: nowMs },
|
|
7371
|
+
demand: tabRequestCost(activeTab),
|
|
7372
|
+
});
|
|
7373
|
+
if (!registered.ok) return null;
|
|
7374
|
+
registeredScopeHash = nextHash;
|
|
7375
|
+
governorRef.current = { scope, leaseId };
|
|
7376
|
+
return scope;
|
|
7377
|
+
}
|
|
7378
|
+
|
|
7379
|
+
function armFromState(nowMs = Date.now()) {
|
|
7380
|
+
if (!scope) return;
|
|
7381
|
+
if (!registeredScopeHash) {
|
|
7382
|
+
armWake("control", governorControlRetryAt(nowMs, runtime.refreshMs), controlWake);
|
|
7383
|
+
return;
|
|
7384
|
+
}
|
|
7385
|
+
const snapshot = inspectGovernor(scope, nowMs);
|
|
7386
|
+
if (!snapshot.ok) {
|
|
7387
|
+
pauseCoordination(TABS[activeIndexRef.current].key, snapshot.reason);
|
|
7388
|
+
armWake("control", governorControlRetryAt(nowMs, runtime.refreshMs), controlWake);
|
|
7389
|
+
return;
|
|
7390
|
+
}
|
|
7391
|
+
publishGovernorEpochs(snapshot);
|
|
7392
|
+
const wakes = governorWakeTimes(snapshot.value, nowMs, runtime.refreshMs, leaseId);
|
|
7393
|
+
armWake("control", wakes.controlAt, controlWake);
|
|
7394
|
+
const nextDataAt = Math.min(wakes.dataAt, activePollAt, backgroundPollAt);
|
|
7395
|
+
if (liveScheduling) armWake("data", nextDataAt, dataWake);
|
|
7396
|
+
}
|
|
7397
|
+
|
|
7398
|
+
function finishPending(key, intentId) {
|
|
7399
|
+
if (pending.get(key)?.intentId === intentId) pending.delete(key);
|
|
7400
|
+
armFromState();
|
|
7401
|
+
}
|
|
7402
|
+
|
|
7403
|
+
function requestTab(key, kind = "active", { force = false } = {}) {
|
|
4823
7404
|
const signal = controller.signal;
|
|
4824
7405
|
const descriptor = tabForKey(key);
|
|
4825
|
-
const
|
|
4826
|
-
if (
|
|
4827
|
-
|
|
4828
|
-
|
|
4829
|
-
|
|
4830
|
-
|
|
7406
|
+
const monotonicNow = performance.now();
|
|
7407
|
+
if (pendingBlockPublications.size > 0) {
|
|
7408
|
+
pauseCoordination(key, "block-unpublished");
|
|
7409
|
+
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7410
|
+
}
|
|
7411
|
+
if (inFlightRef.current[key]) return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7412
|
+
if (!force && backoffActive(`tab:${key}`, monotonicNow)) {
|
|
7413
|
+
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7414
|
+
}
|
|
7415
|
+
if (!force && key === "security" && monotonicNow < securityNextPollAt) {
|
|
7416
|
+
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7417
|
+
}
|
|
7418
|
+
const nowMs = Date.now();
|
|
7419
|
+
const currentScope = ensureScope(nowMs);
|
|
7420
|
+
if (!currentScope) {
|
|
7421
|
+
pauseCoordination(key, "unknown-scope");
|
|
7422
|
+
return Promise.resolve({ persisted: false, retry: true, kind });
|
|
7423
|
+
}
|
|
7424
|
+
const intentGate = runtimeIntentGate(liveScheduling, { force });
|
|
7425
|
+
if (!intentGate.registerIntent) {
|
|
7426
|
+
setTabGovernorDecision(key, { mode: "waiting", probing: true });
|
|
7427
|
+
setTabWaiting(key, true);
|
|
7428
|
+
if (intentGate.requestProbe) {
|
|
7429
|
+
const snapshot = inspectGovernor(currentScope, nowMs);
|
|
7430
|
+
const costs = tabRequestCost(key);
|
|
7431
|
+
for (const resource of RATE_RESOURCES) {
|
|
7432
|
+
const budget = snapshot.value?.budgets?.[resource];
|
|
7433
|
+
if (costs[resource] > 0 && budget) {
|
|
7434
|
+
requestManualProbe(currentScope, leaseId, budget.epoch, budget.observedAt, nowMs);
|
|
7435
|
+
}
|
|
7436
|
+
}
|
|
7437
|
+
}
|
|
7438
|
+
return Promise.resolve({ persisted: false, retry: false, kind });
|
|
7439
|
+
}
|
|
7440
|
+
const existing = pending.get(key);
|
|
7441
|
+
if (existing) {
|
|
7442
|
+
if (!force || existing.kind === "manual") {
|
|
7443
|
+
return Promise.resolve({ persisted: true, retry: false, kind });
|
|
7444
|
+
}
|
|
7445
|
+
cancelIntent(currentScope, existing.intentId, nowMs);
|
|
7446
|
+
pending.delete(key);
|
|
7447
|
+
}
|
|
7448
|
+
const intentId = governorId();
|
|
7449
|
+
const request = {
|
|
7450
|
+
id: intentId,
|
|
7451
|
+
leaseId,
|
|
7452
|
+
tab: key,
|
|
7453
|
+
priority: kind,
|
|
7454
|
+
costs: tabRequestCost(key),
|
|
7455
|
+
requestedAt: nowMs,
|
|
7456
|
+
expiresAt: nowMs + GOVERNOR_LEASE_TTL_MS,
|
|
7457
|
+
};
|
|
7458
|
+
const registered = registerIntent(currentScope, request);
|
|
7459
|
+
if (!registered.ok) {
|
|
7460
|
+
pauseCoordination(key, registered.reason);
|
|
7461
|
+
return Promise.resolve({ persisted: false, retry: true, kind });
|
|
7462
|
+
}
|
|
7463
|
+
pending.set(key, { intentId, kind, force, wasDeferred: false });
|
|
7464
|
+
const decision = registered.value;
|
|
7465
|
+
if (decision.status !== "scheduled" || decision.notBefore > nowMs) {
|
|
7466
|
+
const item = pending.get(key);
|
|
7467
|
+
if (item) item.wasDeferred = true;
|
|
7468
|
+
setTabGovernorDecision(key, visibleGovernorDecision(decision));
|
|
7469
|
+
setTabWaiting(key, true);
|
|
7470
|
+
if (force && decision.resource) {
|
|
7471
|
+
const budget = inspectGovernor(currentScope, nowMs).value?.budgets?.[decision.resource];
|
|
7472
|
+
if (budget) requestManualProbe(currentScope, leaseId, budget.epoch, budget.observedAt, nowMs);
|
|
7473
|
+
}
|
|
7474
|
+
armWake("data", decision.retryAt ?? decision.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7475
|
+
armFromState(nowMs);
|
|
7476
|
+
return Promise.resolve({ persisted: true, retry: false, kind });
|
|
7477
|
+
}
|
|
7478
|
+
const started = startReservation(currentScope, decision.reservationId, nowMs);
|
|
7479
|
+
if (!started.ok) {
|
|
7480
|
+
pauseCoordination(key, started.reason);
|
|
7481
|
+
if (pendingFailureIsTerminal(started.reason)) finishPending(key, intentId);
|
|
7482
|
+
else armWake("data", governorControlRetryAt(nowMs, runtime.refreshMs), dataWake);
|
|
7483
|
+
return Promise.resolve({ persisted: true, retry: false, kind });
|
|
7484
|
+
}
|
|
7485
|
+
if (started.value.status !== "started") {
|
|
7486
|
+
const item = pending.get(key);
|
|
7487
|
+
if (item) item.wasDeferred = true;
|
|
7488
|
+
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
7489
|
+
...decision,
|
|
7490
|
+
...started.value,
|
|
7491
|
+
}));
|
|
7492
|
+
armWake("data", started.value.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7493
|
+
return Promise.resolve({ persisted: true, retry: false, kind });
|
|
7494
|
+
}
|
|
7495
|
+
setTabWaiting(key, false);
|
|
7496
|
+
if (force) coordinator.invalidate();
|
|
4831
7497
|
return commit(
|
|
4832
7498
|
key,
|
|
4833
7499
|
() => descriptor.fetch({ signal, runLimit: runLimitRef.current }),
|
|
4834
|
-
{
|
|
4835
|
-
|
|
7500
|
+
{
|
|
7501
|
+
force,
|
|
7502
|
+
scope: currentScope,
|
|
7503
|
+
reservationId: decision.reservationId,
|
|
7504
|
+
automaticStatusVisible: false,
|
|
7505
|
+
onSettled: () => finishPending(key, intentId),
|
|
7506
|
+
},
|
|
7507
|
+
).then(() => ({ persisted: true, retry: false, kind }));
|
|
7508
|
+
}
|
|
7509
|
+
fetchTabRef.current = (key, { force = false, kind = force ? "manual" : "active" } = {}) => {
|
|
7510
|
+
const requestedAt = Date.now();
|
|
7511
|
+
if (kind === "tab-switch") activePollAt = requestedAt + runtime.refreshMs;
|
|
7512
|
+
const currentScope = ensureScope(requestedAt);
|
|
7513
|
+
if (currentScope) {
|
|
7514
|
+
heartbeatLease(currentScope, leaseId, tabRequestCost(key), requestedAt, key);
|
|
7515
|
+
if (kind === "tab-switch") {
|
|
7516
|
+
for (const [pendingKey, item] of [...pending]) {
|
|
7517
|
+
if (pendingKey !== key && ["active", "tab-switch"].includes(item.kind)) {
|
|
7518
|
+
cancelIntent(currentScope, item.intentId, requestedAt);
|
|
7519
|
+
pending.delete(pendingKey);
|
|
7520
|
+
}
|
|
7521
|
+
}
|
|
7522
|
+
}
|
|
7523
|
+
}
|
|
7524
|
+
return requestTab(key, kind, { force });
|
|
7525
|
+
};
|
|
7526
|
+
|
|
7527
|
+
async function resumePending(nowMs) {
|
|
7528
|
+
for (const [key, item] of [...pending]) {
|
|
7529
|
+
if (inFlightRef.current[key]) continue;
|
|
7530
|
+
const currentScope = ensureScope(nowMs);
|
|
7531
|
+
if (!currentScope) continue;
|
|
7532
|
+
const decision = readIntentDecision(currentScope, item.intentId, nowMs);
|
|
7533
|
+
if (!decision.ok) {
|
|
7534
|
+
pauseCoordination(key, decision.reason);
|
|
7535
|
+
if (pendingFailureIsTerminal(decision.reason)) pending.delete(key);
|
|
7536
|
+
else armWake("data", governorControlRetryAt(nowMs, runtime.refreshMs), dataWake);
|
|
7537
|
+
continue;
|
|
7538
|
+
}
|
|
7539
|
+
if (decision.value.status !== "scheduled" || decision.value.notBefore > nowMs) {
|
|
7540
|
+
item.wasDeferred = true;
|
|
7541
|
+
setTabGovernorDecision(key, visibleGovernorDecision(decision.value));
|
|
7542
|
+
setTabWaiting(key, true);
|
|
7543
|
+
armWake("data", decision.value.retryAt ?? decision.value.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7544
|
+
continue;
|
|
7545
|
+
}
|
|
7546
|
+
const started = startReservation(currentScope, decision.value.reservationId, nowMs);
|
|
7547
|
+
if (!started.ok) {
|
|
7548
|
+
pauseCoordination(key, started.reason);
|
|
7549
|
+
if (pendingFailureIsTerminal(started.reason)) pending.delete(key);
|
|
7550
|
+
else armWake("data", governorControlRetryAt(nowMs, runtime.refreshMs), dataWake);
|
|
7551
|
+
armFromState(nowMs);
|
|
7552
|
+
continue;
|
|
7553
|
+
}
|
|
7554
|
+
if (started.value.status !== "started") {
|
|
7555
|
+
item.wasDeferred = true;
|
|
7556
|
+
setTabGovernorDecision(key, visibleGovernorDecision({
|
|
7557
|
+
...decision.value,
|
|
7558
|
+
...started.value,
|
|
7559
|
+
}));
|
|
7560
|
+
armWake("data", started.value.notBefore ?? nowMs + runtime.refreshMs, dataWake);
|
|
7561
|
+
continue;
|
|
7562
|
+
}
|
|
7563
|
+
setTabWaiting(key, false);
|
|
7564
|
+
if (item.force) coordinator.invalidate();
|
|
7565
|
+
const descriptor = tabForKey(key);
|
|
7566
|
+
void commit(
|
|
7567
|
+
key,
|
|
7568
|
+
() => descriptor.fetch({ signal: controller.signal, runLimit: runLimitRef.current }),
|
|
7569
|
+
{
|
|
7570
|
+
force: item.force,
|
|
7571
|
+
scope: currentScope,
|
|
7572
|
+
reservationId: decision.value.reservationId,
|
|
7573
|
+
automaticStatusVisible: item.wasDeferred && !item.force,
|
|
7574
|
+
onSettled: () => finishPending(key, item.intentId),
|
|
7575
|
+
},
|
|
7576
|
+
);
|
|
7577
|
+
}
|
|
4836
7578
|
}
|
|
4837
|
-
|
|
4838
|
-
|
|
4839
|
-
|
|
4840
|
-
// Setup mode owns the whole screen and can only finish by handing off to
|
|
4841
|
-
// gh or exiting. Hidden tab retries cannot make that state more useful,
|
|
4842
|
-
// so stop spawning subprocesses once any endpoint proves there is no
|
|
4843
|
-
// remote. The interval may still wake, but this branch is otherwise idle.
|
|
4844
|
-
if (remoteSetupRef.current) return;
|
|
4845
|
-
// Kicked off, never awaited: the budget probe must not delay the fetches
|
|
4846
|
-
// this tick is for. `probeInFlight` keeps a slow probe from stampeding.
|
|
7579
|
+
|
|
7580
|
+
async function runDataWake() {
|
|
7581
|
+
if (cancelled || remoteSetupRef.current) return;
|
|
4847
7582
|
const nowMs = Date.now();
|
|
4848
|
-
|
|
4849
|
-
|
|
4850
|
-
|
|
4851
|
-
|
|
4852
|
-
|
|
7583
|
+
await resumePending(nowMs);
|
|
7584
|
+
const currentScope = ensureScope(nowMs);
|
|
7585
|
+
const snapshot = currentScope ? inspectGovernor(currentScope, nowMs) : null;
|
|
7586
|
+
if (!snapshot?.ok) return;
|
|
7587
|
+
publishGovernorEpochs(snapshot);
|
|
7588
|
+
const wakes = governorWakeTimes(snapshot.value, nowMs, runtime.refreshMs, leaseId);
|
|
7589
|
+
const heldResources = Object.fromEntries(RATE_RESOURCES.map((resource) => {
|
|
7590
|
+
const budget = snapshot.value.budgets[resource];
|
|
7591
|
+
const decision = budget ? availableForGrant({ budget, nowMs }) : { mode: "paused" };
|
|
7592
|
+
const held = !budget || budget.blockUntil > nowMs || decision.mode !== "open";
|
|
7593
|
+
const retryAt = budget?.blockUntil > nowMs
|
|
7594
|
+
? budget.blockUntil
|
|
7595
|
+
: decision.retryAt ?? wakes.controlAt;
|
|
7596
|
+
return [resource, { held, retryAt }];
|
|
7597
|
+
}));
|
|
4853
7598
|
const active = TABS[activeIndexRef.current].key;
|
|
4854
|
-
const
|
|
4855
|
-
const
|
|
4856
|
-
|
|
4857
|
-
|
|
4858
|
-
|
|
4859
|
-
|
|
4860
|
-
|
|
4861
|
-
|
|
4862
|
-
|
|
4863
|
-
|
|
4864
|
-
|
|
4865
|
-
|
|
4866
|
-
|
|
7599
|
+
const previousBackgroundIndex = backgroundIndex;
|
|
7600
|
+
const planned = pollSchedule({
|
|
7601
|
+
nowMs,
|
|
7602
|
+
floorMs: runtime.refreshMs,
|
|
7603
|
+
activeKey: active,
|
|
7604
|
+
activeAt: activePollAt,
|
|
7605
|
+
backgroundAt: backgroundPollAt,
|
|
7606
|
+
backgroundIndex,
|
|
7607
|
+
heldResources,
|
|
7608
|
+
});
|
|
7609
|
+
activePollAt = planned.activeAt;
|
|
7610
|
+
backgroundPollAt = planned.backgroundAt;
|
|
7611
|
+
backgroundIndex = planned.backgroundIndex;
|
|
7612
|
+
const outcomes = await Promise.allSettled(
|
|
7613
|
+
planned.due.map(({ key, kind }) => requestTab(key, kind)),
|
|
4867
7614
|
);
|
|
7615
|
+
const retryAt = governorControlRetryAt(Date.now(), runtime.refreshMs);
|
|
7616
|
+
let retryNeeded = false;
|
|
7617
|
+
for (const outcome of outcomes) {
|
|
7618
|
+
if (outcome.status !== "fulfilled" || outcome.value?.retry !== true) continue;
|
|
7619
|
+
({ activeAt: activePollAt, backgroundAt: backgroundPollAt, backgroundIndex } =
|
|
7620
|
+
retryPollAfterAdmissionFailure({
|
|
7621
|
+
kind: outcome.value.kind,
|
|
7622
|
+
retryAt,
|
|
7623
|
+
activeAt: activePollAt,
|
|
7624
|
+
backgroundAt: backgroundPollAt,
|
|
7625
|
+
backgroundIndex,
|
|
7626
|
+
previousBackgroundIndex,
|
|
7627
|
+
}));
|
|
7628
|
+
retryNeeded = true;
|
|
7629
|
+
}
|
|
7630
|
+
if (retryNeeded) armWake("data", retryAt, dataWake);
|
|
7631
|
+
if (!cancelled) {
|
|
7632
|
+
setNow((prev) =>
|
|
7633
|
+
hasInProgressRef.current || Date.now() - prev.getTime() >= 60_000 ? new Date() : prev,
|
|
7634
|
+
);
|
|
7635
|
+
}
|
|
7636
|
+
}
|
|
7637
|
+
|
|
7638
|
+
// A reset can make the old reservation wake and the control wake due in
|
|
7639
|
+
// the same turn. Both may arm `data` before either async callback reaches
|
|
7640
|
+
// pollSchedule(). Let one callback own that transition; the settled hook
|
|
7641
|
+
// retains every later useful wake. Without this guard one pane could
|
|
7642
|
+
// consume two freshly opened lane slots while another pane received none.
|
|
7643
|
+
const dataWake = createSingleFlightWake(runDataWake, () => {
|
|
7644
|
+
if (!cancelled) armFromState(Date.now());
|
|
7645
|
+
});
|
|
7646
|
+
|
|
7647
|
+
async function controlWake() {
|
|
7648
|
+
if (cancelled) return;
|
|
7649
|
+
const nowMs = Date.now();
|
|
7650
|
+
const currentScope = ensureScope(nowMs, { maintain: true });
|
|
7651
|
+
if (currentScope && pendingBlockPublications.size > 0) {
|
|
7652
|
+
const beforeProbe = inspectGovernor(currentScope, nowMs);
|
|
7653
|
+
attemptPendingBlockPublications(
|
|
7654
|
+
currentScope,
|
|
7655
|
+
nowMs,
|
|
7656
|
+
beforeProbe.ok ? beforeProbe.value : null,
|
|
7657
|
+
true,
|
|
7658
|
+
);
|
|
7659
|
+
}
|
|
7660
|
+
const refreshed = currentScope
|
|
7661
|
+
? await refreshSharedBudget(currentScope, leaseId, controller.signal)
|
|
7662
|
+
: { ok: false, reason: "stale" };
|
|
7663
|
+
if (cancelled) return;
|
|
7664
|
+
const checkedAt = Date.now();
|
|
7665
|
+
const snapshot = currentScope ? inspectGovernor(currentScope, checkedAt) : refreshed;
|
|
7666
|
+
const activeKey = TABS[activeIndexRef.current].key;
|
|
7667
|
+
if (currentScope && snapshot.ok && pendingBlockPublications.size > 0) {
|
|
7668
|
+
attemptPendingBlockPublications(currentScope, checkedAt, snapshot.value);
|
|
7669
|
+
}
|
|
7670
|
+
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
7671
|
+
if (pendingBlockPublications.size === 0 && !liveScheduling && governorDataReady(
|
|
7672
|
+
refreshed,
|
|
7673
|
+
snapshot,
|
|
7674
|
+
activeKey,
|
|
7675
|
+
checkedAt,
|
|
7676
|
+
)) {
|
|
7677
|
+
liveScheduling = true;
|
|
7678
|
+
activePollAt = checkedAt + 1;
|
|
7679
|
+
backgroundPollAt = checkedAt + 4 * runtime.refreshMs;
|
|
7680
|
+
armWake("heartbeat", checkedAt + GOVERNOR_HEARTBEAT_MS, heartbeatWake);
|
|
7681
|
+
}
|
|
7682
|
+
if (!refreshed.ok) {
|
|
7683
|
+
armWake("control", governorControlRetryAt(checkedAt, runtime.refreshMs), controlWake);
|
|
7684
|
+
}
|
|
7685
|
+
if (pendingBlockPublications.size > 0) {
|
|
7686
|
+
armWake("control", governorControlRetryAt(checkedAt, runtime.refreshMs), controlWake);
|
|
7687
|
+
}
|
|
7688
|
+
setNow((previous) => checkedAt - previous.getTime() >= 60_000 ? new Date(checkedAt) : previous);
|
|
7689
|
+
armFromState(checkedAt);
|
|
4868
7690
|
}
|
|
4869
7691
|
|
|
4870
|
-
|
|
4871
|
-
|
|
4872
|
-
|
|
4873
|
-
|
|
4874
|
-
|
|
4875
|
-
|
|
4876
|
-
|
|
4877
|
-
let lastProbeAt = 0;
|
|
4878
|
-
let probeInFlight = false;
|
|
4879
|
-
// The open probe window, and the last share it was able to measure. Both are
|
|
4880
|
-
// carried by nextProbeWindow/inferShare rather than reset every probe -- see
|
|
4881
|
-
// those functions for why a throttled pane must not re-measure from scratch.
|
|
4882
|
-
const probeWindows = { core: null, graphql: null };
|
|
4883
|
-
const lastShares = { core: 1, graphql: 1 };
|
|
4884
|
-
let resourceTargets = { core: runtime.refreshMs, graphql: runtime.refreshMs };
|
|
4885
|
-
|
|
4886
|
-
function rearm(ms) {
|
|
4887
|
-
appliedMs = ms;
|
|
4888
|
-
if (intervalId) clearInterval(intervalId);
|
|
4889
|
-
intervalId = setInterval(tick, ms);
|
|
7692
|
+
function heartbeatWake() {
|
|
7693
|
+
if (cancelled) return;
|
|
7694
|
+
const nowMs = Date.now();
|
|
7695
|
+
const currentScope = ensureScope(nowMs);
|
|
7696
|
+
const activeTab = TABS[activeIndexRef.current].key;
|
|
7697
|
+
if (currentScope) heartbeatLease(currentScope, leaseId, tabRequestCost(activeTab), nowMs, activeTab);
|
|
7698
|
+
armWake("heartbeat", nowMs + GOVERNOR_HEARTBEAT_MS, heartbeatWake);
|
|
4890
7699
|
}
|
|
4891
7700
|
|
|
4892
|
-
async function
|
|
4893
|
-
|
|
4894
|
-
|
|
4895
|
-
|
|
4896
|
-
|
|
4897
|
-
|
|
4898
|
-
|
|
4899
|
-
|
|
4900
|
-
|
|
4901
|
-
for (const resource of ["core", "graphql"]) {
|
|
4902
|
-
const budget = budgets[resource];
|
|
4903
|
-
if (!budget) continue;
|
|
4904
|
-
const step = nextProbeWindow(probeWindows[resource], budget, spentTotal[resource]);
|
|
4905
|
-
probeWindows[resource] = step.next;
|
|
4906
|
-
samples[resource] = step.sample;
|
|
4907
|
-
lastShares[resource] = inferShare(step.sample, lastShares[resource]);
|
|
4908
|
-
}
|
|
4909
|
-
const nextTargets = nextBudgetTargets({
|
|
4910
|
-
budgets,
|
|
4911
|
-
samples,
|
|
4912
|
-
shares: lastShares,
|
|
4913
|
-
previous: resourceTargets,
|
|
4914
|
-
activeKey: TABS[activeIndexRef.current].key,
|
|
4915
|
-
floorMs: runtime.refreshMs,
|
|
4916
|
-
nowMs: Date.now(),
|
|
4917
|
-
});
|
|
4918
|
-
resourceTargets = nextTargets.targets;
|
|
4919
|
-
const target = nextTargets.targetMs;
|
|
4920
|
-
if (adaptiveChangeWorthApplying(appliedMs, target)) {
|
|
4921
|
-
rearm(target);
|
|
4922
|
-
setThrottleMs(target > runtime.refreshMs ? target : null);
|
|
4923
|
-
}
|
|
4924
|
-
} finally {
|
|
4925
|
-
probeInFlight = false;
|
|
7701
|
+
async function bootstrap() {
|
|
7702
|
+
remoteUrls = await gitRemoteUrls();
|
|
7703
|
+
runtimeRemoteUrls = remoteUrls;
|
|
7704
|
+
if (cancelled) return;
|
|
7705
|
+
const currentScope = ensureScope(Date.now());
|
|
7706
|
+
if (!currentScope) {
|
|
7707
|
+
pauseCoordination(TABS[activeIndexRef.current].key, "unknown-scope");
|
|
7708
|
+
armWake("control", governorControlRetryAt(Date.now(), runtime.refreshMs), controlWake);
|
|
7709
|
+
return;
|
|
4926
7710
|
}
|
|
7711
|
+
const refreshed = await refreshSharedBudget(currentScope, leaseId, controller.signal);
|
|
7712
|
+
if (cancelled) return;
|
|
7713
|
+
const checkedAt = Date.now();
|
|
7714
|
+
const snapshot = inspectGovernor(currentScope, checkedAt);
|
|
7715
|
+
const activeKey = TABS[activeIndexRef.current].key;
|
|
7716
|
+
publishControlStatus(activeKey, refreshed, snapshot, checkedAt);
|
|
7717
|
+
if (governorDataReady(
|
|
7718
|
+
refreshed,
|
|
7719
|
+
snapshot,
|
|
7720
|
+
activeKey,
|
|
7721
|
+
checkedAt,
|
|
7722
|
+
)) {
|
|
7723
|
+
liveScheduling = true;
|
|
7724
|
+
activePollAt = checkedAt + 1;
|
|
7725
|
+
backgroundPollAt = checkedAt + 4 * runtime.refreshMs;
|
|
7726
|
+
armWake("heartbeat", checkedAt + GOVERNOR_HEARTBEAT_MS, heartbeatWake);
|
|
7727
|
+
}
|
|
7728
|
+
if (!refreshed.ok) {
|
|
7729
|
+
armWake("control", governorControlRetryAt(checkedAt, runtime.refreshMs), controlWake);
|
|
7730
|
+
}
|
|
7731
|
+
armFromState(checkedAt);
|
|
4927
7732
|
}
|
|
4928
7733
|
|
|
4929
|
-
|
|
4930
|
-
rearm(runtime.refreshMs);
|
|
7734
|
+
void bootstrap();
|
|
4931
7735
|
return () => {
|
|
4932
7736
|
cancelled = true;
|
|
4933
|
-
|
|
7737
|
+
wakeScheduler.clearAll();
|
|
7738
|
+
for (const item of pending.values()) cancelIntent(cleanupScope, item.intentId, Date.now());
|
|
7739
|
+
if (cleanupScope && registeredScopeHash) releaseLease(cleanupScope, leaseId);
|
|
7740
|
+
if (governorRef.current?.leaseId === leaseId) governorRef.current = null;
|
|
4934
7741
|
if (contextCoordinatorRef.current === coordinator) contextCoordinatorRef.current = null;
|
|
4935
7742
|
// `cancelled` stops state updates from a promise that already resolved;
|
|
4936
7743
|
// the signal stops the subprocess itself, so quitting doesn't orphan up to
|
|
@@ -4938,14 +7745,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4938
7745
|
// needed.
|
|
4939
7746
|
controller.abort();
|
|
4940
7747
|
};
|
|
4941
|
-
}, [dashboardCacheWriter]);
|
|
7748
|
+
}, [dashboardCacheWriter, screenReader]);
|
|
4942
7749
|
|
|
4943
7750
|
// Background tabs can be up to BACKGROUND_EVERY ticks stale, so the tab you
|
|
4944
7751
|
// switch to refreshes straight away rather than showing old data until its
|
|
4945
7752
|
// slot next comes round. On mount this is a no-op: the initial tick already
|
|
4946
7753
|
// has every tab in flight, and the per-tab guard rejects the duplicate.
|
|
4947
7754
|
useEffect(() => {
|
|
4948
|
-
fetchTabRef.current?.(TABS[activeIndex].key);
|
|
7755
|
+
fetchTabRef.current?.(TABS[activeIndex].key, { kind: "tab-switch" });
|
|
4949
7756
|
}, [activeIndex]);
|
|
4950
7757
|
|
|
4951
7758
|
// Animate only when something on screen is genuinely moving. Automatic polls
|
|
@@ -4964,10 +7771,22 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
4964
7771
|
// indefinitely, on the single most ordinary failure there is. Motion now means
|
|
4965
7772
|
// "still working"; the error line means "not working"; nothing claims both.
|
|
4966
7773
|
const firstLoad = Object.fromEntries(
|
|
4967
|
-
TABS.map((t) => [t.key, data[t.key] == null && !errors[t.key]]),
|
|
7774
|
+
TABS.map((t) => [t.key, data[t.key] == null && !errors[t.key] && loading[t.key]]),
|
|
4968
7775
|
);
|
|
4969
7776
|
const anyFirstLoad = Object.values(firstLoad).some(Boolean);
|
|
4970
|
-
const
|
|
7777
|
+
const semanticStatus = refreshStatus({
|
|
7778
|
+
widthMode: resizeRef.current.active,
|
|
7779
|
+
remoteSetup,
|
|
7780
|
+
visibleLoading: Boolean(loading[tab.key]),
|
|
7781
|
+
visibleInFlight: Boolean(activeRequestStatus),
|
|
7782
|
+
automaticStatusVisible: Boolean(activeRequestStatus?.automaticStatusVisible),
|
|
7783
|
+
governorDecision: activeGovernorDecision,
|
|
7784
|
+
activeError: tabError,
|
|
7785
|
+
securityIncomplete:
|
|
7786
|
+
tab.key === "security" && securityBlind,
|
|
7787
|
+
screenReader,
|
|
7788
|
+
});
|
|
7789
|
+
const showSpinner = !remoteSetup && ANIMATE && (semanticStatus.animate || hasRunningVisible);
|
|
4971
7790
|
useEffect(() => {
|
|
4972
7791
|
if (!showSpinner) {
|
|
4973
7792
|
// Park on a fixed frame rather than freezing wherever the animation
|
|
@@ -5045,9 +7864,14 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
5045
7864
|
// its own and the label is minute-granular by design.
|
|
5046
7865
|
const lastOk = lastOkRef.current[tab.key];
|
|
5047
7866
|
const staleFor = lastOk == null ? null : now.getTime() - lastOk;
|
|
5048
|
-
const
|
|
7867
|
+
const staleAt = freshnessDeadline({
|
|
7868
|
+
lastOk,
|
|
7869
|
+
refreshMs: runtime.refreshMs,
|
|
7870
|
+
governorDecision: activeGovernorDecision,
|
|
7871
|
+
currentEpochs: governorEpochs,
|
|
7872
|
+
});
|
|
5049
7873
|
const staleLabel =
|
|
5050
|
-
staleFor != null &&
|
|
7874
|
+
staleFor != null && staleAt != null && now.getTime() > staleAt
|
|
5051
7875
|
? `stale ${formatDuration(Math.min(staleFor, 359_999_000))}`
|
|
5052
7876
|
: null;
|
|
5053
7877
|
|
|
@@ -5251,6 +8075,15 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
5251
8075
|
!showHelp &&
|
|
5252
8076
|
tooNarrow &&
|
|
5253
8077
|
e(Text, { dimColor: true, wrap: "truncate-end" }, "too narrow"),
|
|
8078
|
+
!showHelp &&
|
|
8079
|
+
!tooNarrow &&
|
|
8080
|
+
!remoteSetup &&
|
|
8081
|
+
activeGovernorDecision?.coordinationError &&
|
|
8082
|
+
e(
|
|
8083
|
+
Text,
|
|
8084
|
+
{ color: ATTENTION, wrap: "truncate-end" },
|
|
8085
|
+
`API coordination unavailable (${activeGovernorDecision.reason}); retrying safely`,
|
|
8086
|
+
),
|
|
5254
8087
|
!showHelp &&
|
|
5255
8088
|
!tooNarrow &&
|
|
5256
8089
|
!remoteSetup &&
|
|
@@ -5334,6 +8167,8 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
5334
8167
|
// Suppressed once icons are already ASCII, since then there is
|
|
5335
8168
|
// nothing to fix.
|
|
5336
8169
|
(USING_NERD_ICONS && iconHintDue ? " (icons blank? GH_GLANCE_ICONS=unicode)" : "")
|
|
8170
|
+
: ["waiting", "paused"].includes(semanticStatus.kind) || waiting[tab.key]
|
|
8171
|
+
? "waiting for API budget…"
|
|
5337
8172
|
: tab.key === "security" && securityNotes.length === ALERT_SOURCES.length
|
|
5338
8173
|
? "no alert sources available"
|
|
5339
8174
|
: `no ${tab.countLabel}`,
|
|
@@ -5345,10 +8180,10 @@ function App({ onCreateRemote = () => {} } = {}) {
|
|
|
5345
8180
|
),
|
|
5346
8181
|
e(PanelEdge, { width: frameCols, top: false, label: countLabel, labelColor: BORDER_COLOR }),
|
|
5347
8182
|
e(StatusBar, {
|
|
5348
|
-
|
|
8183
|
+
status: semanticStatus,
|
|
8184
|
+
detail: activeGovernorDecision,
|
|
5349
8185
|
spin: showSpinner ? spin : null,
|
|
5350
8186
|
stale: staleLabel,
|
|
5351
|
-
throttle: throttleMs ? `throttled ${Math.round(throttleMs / 1000)}s` : null,
|
|
5352
8187
|
interactive,
|
|
5353
8188
|
cols: frameCols,
|
|
5354
8189
|
remoteSetup,
|
|
@@ -5616,20 +8451,73 @@ export {
|
|
|
5616
8451
|
ALERT_SOURCES,
|
|
5617
8452
|
REST_PER_FETCH,
|
|
5618
8453
|
GRAPHQL_PER_FETCH,
|
|
8454
|
+
OPERATION_COSTS,
|
|
8455
|
+
operationCost,
|
|
8456
|
+
tabRequestCost,
|
|
5619
8457
|
projectedHourlyCost,
|
|
5620
8458
|
REFRESH_MS,
|
|
5621
8459
|
BACKGROUND_EVERY,
|
|
5622
|
-
|
|
5623
|
-
nextBudgetTargets,
|
|
5624
|
-
adaptiveChangeWorthApplying,
|
|
5625
|
-
sampleIsUsable,
|
|
5626
|
-
inferShare,
|
|
5627
|
-
nextProbeWindow,
|
|
5628
|
-
restPerTick,
|
|
5629
|
-
MAX_ADAPTIVE_REFRESH_MS,
|
|
8460
|
+
externalSampleIsUsable,
|
|
5630
8461
|
BUDGET_SAFETY,
|
|
8462
|
+
BUDGET_RESERVE_FRACTION,
|
|
8463
|
+
BUDGET_SNAPSHOT_TTL_MS,
|
|
8464
|
+
GOVERNOR_HEARTBEAT_MS,
|
|
8465
|
+
GOVERNOR_LEASE_TTL_MS,
|
|
8466
|
+
GOVERNOR_PROBE_LEASE_MS,
|
|
8467
|
+
GOVERNOR_ACTIVE_PROBE_LEASE_MS,
|
|
8468
|
+
BUDGET_RESET_GRACE_MS,
|
|
8469
|
+
GOVERNOR_PHASE_WINDOW_MS,
|
|
8470
|
+
BUDGET_PROBE_MS,
|
|
5631
8471
|
MIN_SAMPLE_CALLS,
|
|
5632
|
-
|
|
8472
|
+
REQUEST_PRIORITIES,
|
|
8473
|
+
normalizeBudgetResource,
|
|
8474
|
+
budgetEpoch,
|
|
8475
|
+
resourceReserve,
|
|
8476
|
+
availableForGrant,
|
|
8477
|
+
nextExternalFactor,
|
|
8478
|
+
resourceDecision,
|
|
8479
|
+
governorPhaseOffset,
|
|
8480
|
+
scheduleIntents,
|
|
8481
|
+
GOVERNOR_STATE_VERSION,
|
|
8482
|
+
GOVERNOR_MAX_LEASES,
|
|
8483
|
+
GOVERNOR_MAX_INTENTS,
|
|
8484
|
+
GOVERNOR_MAX_RESERVATIONS,
|
|
8485
|
+
GOVERNOR_LOCK_WAIT_MS,
|
|
8486
|
+
GOVERNOR_PROBE_DRAIN_MS,
|
|
8487
|
+
GOVERNOR_ID_PATTERN,
|
|
8488
|
+
governorId,
|
|
8489
|
+
normalizeHost,
|
|
8490
|
+
remoteHost,
|
|
8491
|
+
resolveEffectiveHost,
|
|
8492
|
+
governorScopeHash,
|
|
8493
|
+
governorPath,
|
|
8494
|
+
createGovernorScope,
|
|
8495
|
+
emptyGovernorState,
|
|
8496
|
+
normalizeGovernorState,
|
|
8497
|
+
serializeGovernorState,
|
|
8498
|
+
readGovernorState,
|
|
8499
|
+
writeGovernorState,
|
|
8500
|
+
pidIsDead,
|
|
8501
|
+
releaseGovernorLock,
|
|
8502
|
+
withGovernorLock,
|
|
8503
|
+
registerLease,
|
|
8504
|
+
heartbeatLease,
|
|
8505
|
+
maintainControlLease,
|
|
8506
|
+
claimProbe,
|
|
8507
|
+
renewProbeClaim,
|
|
8508
|
+
publishProbe,
|
|
8509
|
+
failProbeClaim,
|
|
8510
|
+
requestManualProbe,
|
|
8511
|
+
registerIntent,
|
|
8512
|
+
readIntentDecision,
|
|
8513
|
+
cancelIntent,
|
|
8514
|
+
startReservation,
|
|
8515
|
+
completeReservation,
|
|
8516
|
+
recordResourceBlock,
|
|
8517
|
+
releaseLease,
|
|
8518
|
+
inspectGovernor,
|
|
8519
|
+
governorHealth,
|
|
8520
|
+
refreshSharedBudget,
|
|
5633
8521
|
safe,
|
|
5634
8522
|
shortErr,
|
|
5635
8523
|
isUnavailable,
|
|
@@ -5642,10 +8530,12 @@ export {
|
|
|
5642
8530
|
parseRepoContext,
|
|
5643
8531
|
parseAuthContext,
|
|
5644
8532
|
buildFailureContext,
|
|
8533
|
+
resolveFailureContext,
|
|
5645
8534
|
failureTargetHost,
|
|
5646
8535
|
unavailableRemedy,
|
|
5647
8536
|
createFailureContextCoordinator,
|
|
5648
8537
|
createOpenRequestRegistry,
|
|
8538
|
+
openInBrowser,
|
|
5649
8539
|
AUTH_RETRY_MS,
|
|
5650
8540
|
BACKOFF_STEPS_MS,
|
|
5651
8541
|
redact,
|
|
@@ -5687,9 +8577,27 @@ export {
|
|
|
5687
8577
|
shouldCheckpointFreshness,
|
|
5688
8578
|
securityPollDelay,
|
|
5689
8579
|
shouldShowFetchLoading,
|
|
5690
|
-
|
|
8580
|
+
pollSchedule,
|
|
8581
|
+
retryPollAfterAdmissionFailure,
|
|
8582
|
+
governorWakeTimes,
|
|
8583
|
+
governorDataReady,
|
|
8584
|
+
governorControlRetryAt,
|
|
8585
|
+
createWakeScheduler,
|
|
8586
|
+
createSingleFlightWake,
|
|
8587
|
+
pendingFailureIsTerminal,
|
|
8588
|
+
runtimeIntentGate,
|
|
8589
|
+
rateLimitBlockDecision,
|
|
8590
|
+
mergeRateLimitBlockPublications,
|
|
8591
|
+
hydrateRateLimitBlockPublication,
|
|
8592
|
+
retryRateLimitBlockPublication,
|
|
8593
|
+
rateLimitBlockProbeRecovered,
|
|
8594
|
+
admitGovernorOperation,
|
|
8595
|
+
runAdmittedOperation,
|
|
5691
8596
|
pollResultTransition,
|
|
5692
8597
|
forcedBackoffKeys,
|
|
8598
|
+
clearForcedBackoffAfterStart,
|
|
8599
|
+
doctorProbePlan,
|
|
8600
|
+
mapAllSettledBounded,
|
|
5693
8601
|
alertRequestArgs,
|
|
5694
8602
|
shouldFetchAlertPriorityLanes,
|
|
5695
8603
|
mergeAlertRows,
|
|
@@ -5703,6 +8611,7 @@ export {
|
|
|
5703
8611
|
TABS,
|
|
5704
8612
|
OCT_NERD,
|
|
5705
8613
|
OCT_UNICODE,
|
|
8614
|
+
normalizeIconProfile,
|
|
5706
8615
|
KEY_TABLE,
|
|
5707
8616
|
KEY_HINTS,
|
|
5708
8617
|
activeKeyHints,
|
|
@@ -5712,6 +8621,12 @@ export {
|
|
|
5712
8621
|
helpLines,
|
|
5713
8622
|
summarizeDoctorEnv,
|
|
5714
8623
|
widthStatusText,
|
|
8624
|
+
refreshStatus,
|
|
8625
|
+
statusBarLayout,
|
|
8626
|
+
freshnessDeadline,
|
|
8627
|
+
probingGovernorDecisions,
|
|
8628
|
+
retainDeferredGovernorHold,
|
|
8629
|
+
REFRESH_STATUS_GLYPHS,
|
|
5715
8630
|
headerGutterKey,
|
|
5716
8631
|
HeaderCells,
|
|
5717
8632
|
parseSgrMouse,
|
|
@@ -5726,6 +8641,5 @@ export {
|
|
|
5726
8641
|
REMOTE_SETUP_LINES,
|
|
5727
8642
|
REMOTE_SETUP_NONINTERACTIVE_LINES,
|
|
5728
8643
|
VERDICT_REMEDY,
|
|
5729
|
-
RATE_LIMIT_RETRY_MS,
|
|
5730
8644
|
FAILURE_LADDER,
|
|
5731
8645
|
};
|